// Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Based on CRYPTOGAMS code with the following comment: // # ==================================================================== // # Written by Andy Polyakov for the OpenSSL // # project. The module is, however, dual licensed under OpenSSL and // # CRYPTOGAMS licenses depending on where you obtain it. For further // # details see http://www.openssl.org/~appro/cryptogams/. // # ==================================================================== // Code for the perl script that generates the ppc64 assembler // can be found in the cryptogams repository at the link below. It is based on // the original from openssl. // https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91 // The differences in this and the original implementation are // due to the calling conventions and initialization of constants. // +build !gccgo,!purego #include "textflag.h" #define OUT R3 #define INP R4 #define LEN R5 #define KEY R6 #define CNT R7 #define TMP R15 #define CONSTBASE R16 #define BLOCKS R17 DATA consts<>+0x00(SB)/8, $0x3320646e61707865 DATA consts<>+0x08(SB)/8, $0x6b20657479622d32 DATA consts<>+0x10(SB)/8, $0x0000000000000001 DATA consts<>+0x18(SB)/8, $0x0000000000000000 DATA consts<>+0x20(SB)/8, $0x0000000000000004 DATA consts<>+0x28(SB)/8, $0x0000000000000000 DATA consts<>+0x30(SB)/8, $0x0a0b08090e0f0c0d DATA consts<>+0x38(SB)/8, $0x0203000106070405 DATA consts<>+0x40(SB)/8, $0x090a0b080d0e0f0c DATA consts<>+0x48(SB)/8, $0x0102030005060704 DATA consts<>+0x50(SB)/8, $0x6170786561707865 DATA consts<>+0x58(SB)/8, $0x6170786561707865 DATA consts<>+0x60(SB)/8, $0x3320646e3320646e DATA consts<>+0x68(SB)/8, $0x3320646e3320646e DATA consts<>+0x70(SB)/8, $0x79622d3279622d32 DATA consts<>+0x78(SB)/8, $0x79622d3279622d32 DATA consts<>+0x80(SB)/8, $0x6b2065746b206574 DATA consts<>+0x88(SB)/8, $0x6b2065746b206574 DATA consts<>+0x90(SB)/8, $0x0000000100000000 DATA consts<>+0x98(SB)/8, $0x0000000300000002 GLOBL consts<>(SB), RODATA, $0xa0 //func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32) TEXT ·chaCha20_ctr32_vsx(SB),NOSPLIT,$64-40 MOVD out+0(FP), OUT MOVD inp+8(FP), INP MOVD len+16(FP), LEN MOVD key+24(FP), KEY MOVD counter+32(FP), CNT // Addressing for constants MOVD $consts<>+0x00(SB), CONSTBASE MOVD $16, R8 MOVD $32, R9 MOVD $48, R10 MOVD $64, R11 SRD $6, LEN, BLOCKS // V16 LXVW4X (CONSTBASE)(R0), VS48 ADD $80,CONSTBASE // Load key into V17,V18 LXVW4X (KEY)(R0), VS49 LXVW4X (KEY)(R8), VS50 // Load CNT, NONCE into V19 LXVW4X (CNT)(R0), VS51 // Clear V27 VXOR V27, V27, V27 // V28 LXVW4X (CONSTBASE)(R11), VS60 // splat slot from V19 -> V26 VSPLTW $0, V19, V26 VSLDOI $4, V19, V27, V19 VSLDOI $12, V27, V19, V19 VADDUWM V26, V28, V26 MOVD $10, R14 MOVD R14, CTR loop_outer_vsx: // V0, V1, V2, V3 LXVW4X (R0)(CONSTBASE), VS32 LXVW4X (R8)(CONSTBASE), VS33 LXVW4X (R9)(CONSTBASE), VS34 LXVW4X (R10)(CONSTBASE), VS35 // splat values from V17, V18 into V4-V11 VSPLTW $0, V17, V4 VSPLTW $1, V17, V5 VSPLTW $2, V17, V6 VSPLTW $3, V17, V7 VSPLTW $0, V18, V8 VSPLTW $1, V18, V9 VSPLTW $2, V18, V10 VSPLTW $3, V18, V11 // VOR VOR V26, V26, V12 // splat values from V19 -> V13, V14, V15 VSPLTW $1, V19, V13 VSPLTW $2, V19, V14 VSPLTW $3, V19, V15 // splat const values VSPLTISW $-16, V27 VSPLTISW $12, V28 VSPLTISW $8, V29 VSPLTISW $7, V30 loop_vsx: VADDUWM V0, V4, V0 VADDUWM V1, V5, V1 VADDUWM V2, V6, V2 VADDUWM V3, V7, V3 VXOR V12, V0, V12 VXOR V13, V1, V13 VXOR V14, V2, V14 VXOR V15, V3, V15 VRLW V12, V27, V12 VRLW V13, V27, V13 VRLW V14, V27, V14 VRLW V15, V27, V15 VADDUWM V8, V12, V8 VADDUWM V9, V13, V9 VADDUWM V10, V14, V10 VADDUWM V11, V15, V11 VXOR V4, V8, V4 VXOR V5, V9, V5 VXOR V6, V10, V6 VXOR V7, V11, V7 VRLW V4, V28, V4 VRLW V5, V28, V5 VRLW V6, V28, V6 VRLW V7, V28, V7 VADDUWM V0, V4, V0 VADDUWM V1, V5, V1 VADDUWM V2, V6, V2 VADDUWM V3, V7, V3 VXOR V12, V0, V12 VXOR V13, V1, V13 VXOR V14, V2, V14 VXOR V15, V3, V15 VRLW V12, V29, V12 VRLW V13, V29, V13 VRLW V14, V29, V14 VRLW V15, V29, V15 VADDUWM V8, V12, V8 VADDUWM V9, V13, V9 VADDUWM V10, V14, V10 VADDUWM V11, V15, V11 VXOR V4, V8, V4 VXOR V5, V9, V5 VXOR V6, V10, V6 VXOR V7, V11, V7 VRLW V4, V30, V4 VRLW V5, V30, V5 VRLW V6, V30, V6 VRLW V7, V30, V7 VADDUWM V0, V5, V0 VADDUWM V1, V6, V1 VADDUWM V2, V7, V2 VADDUWM V3, V4, V3 VXOR V15, V0, V15 VXOR V12, V1, V12 VXOR V13, V2, V13 VXOR V14, V3, V14 VRLW V15, V27, V15 VRLW V12, V27, V12 VRLW V13, V27, V13 VRLW V14, V27, V14 VADDUWM V10, V15, V10 VADDUWM V11, V12, V11 VADDUWM V8, V13, V8 VADDUWM V9, V14, V9 VXOR V5, V10, V5 VXOR V6, V11, V6 VXOR V7, V8, V7 VXOR V4, V9, V4 VRLW V5, V28, V5 VRLW V6, V28, V6 VRLW V7, V28, V7 VRLW V4, V28, V4 VADDUWM V0, V5, V0 VADDUWM V1, V6, V1 VADDUWM V2, V7, V2 VADDUWM V3, V4, V3 VXOR V15, V0, V15 VXOR V12, V1, V12 VXOR V13, V2, V13 VXOR V14, V3, V14 VRLW V15, V29, V15 VRLW V12, V29, V12 VRLW V13, V29, V13 VRLW V14, V29, V14 VADDUWM V10, V15, V10 VADDUWM V11, V12, V11 VADDUWM V8, V13, V8 VADDUWM V9, V14, V9 VXOR V5, V10, V5 VXOR V6, V11, V6 VXOR V7, V8, V7 VXOR V4, V9, V4 VRLW V5, V30, V5 VRLW V6, V30, V6 VRLW V7, V30, V7 VRLW V4, V30, V4 BC 16, LT, loop_vsx VADDUWM V12, V26, V12 WORD $0x13600F8C // VMRGEW V0, V1, V27 WORD $0x13821F8C // VMRGEW V2, V3, V28 WORD $0x10000E8C // VMRGOW V0, V1, V0 WORD $0x10421E8C // VMRGOW V2, V3, V2 WORD $0x13A42F8C // VMRGEW V4, V5, V29 WORD $0x13C63F8C // VMRGEW V6, V7, V30 XXPERMDI VS32, VS34, $0, VS33 XXPERMDI VS32, VS34, $3, VS35 XXPERMDI VS59, VS60, $0, VS32 XXPERMDI VS59, VS60, $3, VS34 WORD $0x10842E8C // VMRGOW V4, V5, V4 WORD $0x10C63E8C // VMRGOW V6, V7, V6 WORD $0x13684F8C // VMRGEW V8, V9, V27 WORD $0x138A5F8C // VMRGEW V10, V11, V28 XXPERMDI VS36, VS38, $0, VS37 XXPERMDI VS36, VS38, $3, VS39 XXPERMDI VS61, VS62, $0, VS36 XXPERMDI VS61, VS62, $3, VS38 WORD $0x11084E8C // VMRGOW V8, V9, V8 WORD $0x114A5E8C // VMRGOW V10, V11, V10 WORD $0x13AC6F8C // VMRGEW V12, V13, V29 WORD $0x13CE7F8C // VMRGEW V14, V15, V30 XXPERMDI VS40, VS42, $0, VS41 XXPERMDI VS40, VS42, $3, VS43 XXPERMDI VS59, VS60, $0, VS40 XXPERMDI VS59, VS60, $3, VS42 WORD $0x118C6E8C // VMRGOW V12, V13, V12 WORD $0x11CE7E8C // VMRGOW V14, V15, V14 VSPLTISW $4, V27 VADDUWM V26, V27, V26 XXPERMDI VS44, VS46, $0, VS45 XXPERMDI VS44, VS46, $3, VS47 XXPERMDI VS61, VS62, $0, VS44 XXPERMDI VS61, VS62, $3, VS46 VADDUWM V0, V16, V0 VADDUWM V4, V17, V4 VADDUWM V8, V18, V8 VADDUWM V12, V19, V12 CMPU LEN, $64 BLT tail_vsx // Bottom of loop LXVW4X (INP)(R0), VS59 LXVW4X (INP)(R8), VS60 LXVW4X (INP)(R9), VS61 LXVW4X (INP)(R10), VS62 VXOR V27, V0, V27 VXOR V28, V4, V28 VXOR V29, V8, V29 VXOR V30, V12, V30 STXVW4X VS59, (OUT)(R0) STXVW4X VS60, (OUT)(R8) ADD $64, INP STXVW4X VS61, (OUT)(R9) ADD $-64, LEN STXVW4X VS62, (OUT)(R10) ADD $64, OUT BEQ done_vsx VADDUWM V1, V16, V0 VADDUWM V5, V17, V4 VADDUWM V9, V18, V8 VADDUWM V13, V19, V12 CMPU LEN, $64 BLT tail_vsx LXVW4X (INP)(R0), VS59 LXVW4X (INP)(R8), VS60 LXVW4X (INP)(R9), VS61 LXVW4X (INP)(R10), VS62 VXOR V27, V0, V27 VXOR V28, V4, V28 VXOR V29, V8, V29 VXOR V30, V12, V30 STXVW4X VS59, (OUT)(R0) STXVW4X VS60, (OUT)(R8) ADD $64, INP STXVW4X VS61, (OUT)(R9) ADD $-64, LEN STXVW4X VS62, (OUT)(V10) ADD $64, OUT BEQ done_vsx VADDUWM V2, V16, V0 VADDUWM V6, V17, V4 VADDUWM V10, V18, V8 VADDUWM V14, V19, V12 CMPU LEN, $64 BLT tail_vsx LXVW4X (INP)(R0), VS59 LXVW4X (INP)(R8), VS60 LXVW4X (INP)(R9), VS61 LXVW4X (INP)(R10), VS62 VXOR V27, V0, V27 VXOR V28, V4, V28 VXOR V29, V8, V29 VXOR V30, V12, V30 STXVW4X VS59, (OUT)(R0) STXVW4X VS60, (OUT)(R8) ADD $64, INP STXVW4X VS61, (OUT)(R9) ADD $-64, LEN STXVW4X VS62, (OUT)(R10) ADD $64, OUT BEQ done_vsx VADDUWM V3, V16, V0 VADDUWM V7, V17, V4 VADDUWM V11, V18, V8 VADDUWM V15, V19, V12 CMPU LEN, $64 BLT tail_vsx LXVW4X (INP)(R0), VS59 LXVW4X (INP)(R8), VS60 LXVW4X (INP)(R9), VS61 LXVW4X (INP)(R10), VS62 VXOR V27, V0, V27 VXOR V28, V4, V28 VXOR V29, V8, V29 VXOR V30, V12, V30 STXVW4X VS59, (OUT)(R0) STXVW4X VS60, (OUT)(R8) ADD $64, INP STXVW4X VS61, (OUT)(R9) ADD $-64, LEN STXVW4X VS62, (OUT)(R10) ADD $64, OUT MOVD $10, R14 MOVD R14, CTR BNE loop_outer_vsx done_vsx: // Increment counter by number of 64 byte blocks MOVD (CNT), R14 ADD BLOCKS, R14 MOVD R14, (CNT) RET tail_vsx: ADD $32, R1, R11 MOVD LEN, CTR // Save values on stack to copy from STXVW4X VS32, (R11)(R0) STXVW4X VS36, (R11)(R8) STXVW4X VS40, (R11)(R9) STXVW4X VS44, (R11)(R10) ADD $-1, R11, R12 ADD $-1, INP ADD $-1, OUT looptail_vsx: // Copying the result to OUT // in bytes. MOVBZU 1(R12), KEY MOVBZU 1(INP), TMP XOR KEY, TMP, KEY MOVBU KEY, 1(OUT) BC 16, LT, looptail_vsx // Clear the stack values STXVW4X VS48, (R11)(R0) STXVW4X VS48, (R11)(R8) STXVW4X VS48, (R11)(R9) STXVW4X VS48, (R11)(R10) BR done_vsx