| // Copyright 2019 The Go Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style | |
| // license that can be found in the LICENSE file. | |
| // Based on CRYPTOGAMS code with the following comment: | |
| // # ==================================================================== | |
| // # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL | |
| // # project. The module is, however, dual licensed under OpenSSL and | |
| // # CRYPTOGAMS licenses depending on where you obtain it. For further | |
| // # details see http://www.openssl.org/~appro/cryptogams/. | |
| // # ==================================================================== | |
| // This implementation is based on the ppc64 asm generated by the | |
| // script https://github.com/dot-asm/cryptogams/blob/master/ppc/ghashp8-ppc.pl | |
| // from commit d47afb3c. | |
| // Changes were made due to differences in the ABI and some register usage. | |
| // Some arguments were changed due to the way the Go code passes them. | |
| #include "textflag.h" | |
| #define XIP R3 | |
| #define HTBL R4 | |
| #define INP R5 | |
| #define LEN R6 | |
| #define XL V0 | |
| #define XM V1 | |
| #define XH V2 | |
| #define IN V3 | |
| #define ZERO V4 | |
| #define T0 V5 | |
| #define T1 V6 | |
| #define T2 V7 | |
| #define XC2 V8 | |
| #define H V9 | |
| #define HH V10 | |
| #define HL V11 | |
| #define LEMASK V12 | |
| #define XL1 V13 | |
| #define XM1 V14 | |
| #define XH1 V15 | |
| #define IN1 V16 | |
| #define H2 V17 | |
| #define H2H V18 | |
| #define H2L V19 | |
| #define XL3 V20 | |
| #define XM2 V21 | |
| #define IN2 V22 | |
| #define H3L V23 | |
| #define H3 V24 | |
| #define H3H V25 | |
| #define XH3 V26 | |
| #define XM3 V27 | |
| #define IN3 V28 | |
| #define H4L V29 | |
| #define H4 V30 | |
| #define H4H V31 | |
| #define IN0 IN | |
| #define H21L HL | |
| #define H21H HH | |
| #define LOPERM H2L | |
| #define HIPERM H2H | |
| #define VXL VS32 | |
| #define VIN VS35 | |
| #define VXC2 VS40 | |
| #define VH VS41 | |
| #define VHH VS42 | |
| #define VHL VS43 | |
| #define VIN1 VS48 | |
| #define VH2 VS49 | |
| #define VH2H VS50 | |
| #define VH2L VS51 | |
| #define VIN2 VS54 | |
| #define VH3L VS55 | |
| #define VH3 VS56 | |
| #define VH3H VS57 | |
| #define VIN3 VS60 | |
| #define VH4L VS61 | |
| #define VH4 VS62 | |
| #define VH4H VS63 | |
| #define VIN0 VIN | |
| // func gcmInit(productTable *[256]byte, h []byte) | |
| TEXT ·gcmInit(SB), NOSPLIT, $0-32 | |
| MOVD productTable+0(FP), XIP | |
| MOVD h+8(FP), HTBL | |
| MOVD $0x10, R8 | |
| MOVD $0x20, R9 | |
| MOVD $0x30, R10 | |
| LXVD2X (HTBL)(R0), VH // Load H | |
| VSPLTISB $-16, XC2 // 0xf0 | |
| VSPLTISB $1, T0 // one | |
| VADDUBM XC2, XC2, XC2 // 0xe0 | |
| VXOR ZERO, ZERO, ZERO | |
| VOR XC2, T0, XC2 // 0xe1 | |
| VSLDOI $15, XC2, ZERO, XC2 // 0xe1... | |
| VSLDOI $1, ZERO, T0, T1 // ...1 | |
| VADDUBM XC2, XC2, XC2 // 0xc2... | |
| VSPLTISB $7, T2 | |
| VOR XC2, T1, XC2 // 0xc2....01 | |
| VSPLTB $0, H, T1 // most significant byte | |
| VSL H, T0, H // H<<=1 | |
| VSRAB T1, T2, T1 // broadcast carry bit | |
| VAND T1, XC2, T1 | |
| VXOR H, T1, IN // twisted H | |
| VSLDOI $8, IN, IN, H // twist even more ... | |
| VSLDOI $8, ZERO, XC2, XC2 // 0xc2.0 | |
| VSLDOI $8, ZERO, H, HL // ... and split | |
| VSLDOI $8, H, ZERO, HH | |
| STXVD2X VXC2, (XIP+R0) // save pre-computed table | |
| STXVD2X VHL, (XIP+R8) | |
| MOVD $0x40, R8 | |
| STXVD2X VH, (XIP+R9) | |
| MOVD $0x50, R9 | |
| STXVD2X VHH, (XIP+R10) | |
| MOVD $0x60, R10 | |
| VPMSUMD IN, HL, XL // H.lo·H.lo | |
| VPMSUMD IN, H, XM // H.hi·H.lo+H.lo·H.hi | |
| VPMSUMD IN, HH, XH // H.hi·H.hi | |
| VPMSUMD XL, XC2, T2 // 1st reduction phase | |
| VSLDOI $8, XM, ZERO, T0 | |
| VSLDOI $8, ZERO, XM, T1 | |
| VXOR XL, T0, XL | |
| VXOR XH, T1, XH | |
| VSLDOI $8, XL, XL, XL | |
| VXOR XL, T2, XL | |
| VSLDOI $8, XL, XL, T1 // 2nd reduction phase | |
| VPMSUMD XL, XC2, XL | |
| VXOR T1, XH, T1 | |
| VXOR XL, T1, IN1 | |
| VSLDOI $8, IN1, IN1, H2 | |
| VSLDOI $8, ZERO, H2, H2L | |
| VSLDOI $8, H2, ZERO, H2H | |
| STXVD2X VH2L, (XIP+R8) // save H^2 | |
| MOVD $0x70, R8 | |
| STXVD2X VH2, (XIP+R9) | |
| MOVD $0x80, R9 | |
| STXVD2X VH2H, (XIP+R10) | |
| MOVD $0x90, R10 | |
| VPMSUMD IN, H2L, XL // H.lo·H^2.lo | |
| VPMSUMD IN1, H2L, XL1 // H^2.lo·H^2.lo | |
| VPMSUMD IN, H2, XM // H.hi·H^2.lo+H.lo·H^2.hi | |
| VPMSUMD IN1, H2, XM1 // H^2.hi·H^2.lo+H^2.lo·H^2.hi | |
| VPMSUMD IN, H2H, XH // H.hi·H^2.hi | |
| VPMSUMD IN1, H2H, XH1 // H^2.hi·H^2.hi | |
| VPMSUMD XL, XC2, T2 // 1st reduction phase | |
| VPMSUMD XL1, XC2, HH // 1st reduction phase | |
| VSLDOI $8, XM, ZERO, T0 | |
| VSLDOI $8, ZERO, XM, T1 | |
| VSLDOI $8, XM1, ZERO, HL | |
| VSLDOI $8, ZERO, XM1, H | |
| VXOR XL, T0, XL | |
| VXOR XH, T1, XH | |
| VXOR XL1, HL, XL1 | |
| VXOR XH1, H, XH1 | |
| VSLDOI $8, XL, XL, XL | |
| VSLDOI $8, XL1, XL1, XL1 | |
| VXOR XL, T2, XL | |
| VXOR XL1, HH, XL1 | |
| VSLDOI $8, XL, XL, T1 // 2nd reduction phase | |
| VSLDOI $8, XL1, XL1, H // 2nd reduction phase | |
| VPMSUMD XL, XC2, XL | |
| VPMSUMD XL1, XC2, XL1 | |
| VXOR T1, XH, T1 | |
| VXOR H, XH1, H | |
| VXOR XL, T1, XL | |
| VXOR XL1, H, XL1 | |
| VSLDOI $8, XL, XL, H | |
| VSLDOI $8, XL1, XL1, H2 | |
| VSLDOI $8, ZERO, H, HL | |
| VSLDOI $8, H, ZERO, HH | |
| VSLDOI $8, ZERO, H2, H2L | |
| VSLDOI $8, H2, ZERO, H2H | |
| STXVD2X VHL, (XIP+R8) // save H^3 | |
| MOVD $0xa0, R8 | |
| STXVD2X VH, (XIP+R9) | |
| MOVD $0xb0, R9 | |
| STXVD2X VHH, (XIP+R10) | |
| MOVD $0xc0, R10 | |
| STXVD2X VH2L, (XIP+R8) // save H^4 | |
| STXVD2X VH2, (XIP+R9) | |
| STXVD2X VH2H, (XIP+R10) | |
| RET | |
| // func gcmHash(output []byte, productTable *[256]byte, inp []byte, len int) | |
| TEXT ·gcmHash(SB), NOSPLIT, $0-64 | |
| MOVD output+0(FP), XIP | |
| MOVD productTable+24(FP), HTBL | |
| MOVD inp+32(FP), INP | |
| MOVD len+56(FP), LEN | |
| MOVD $0x10, R8 | |
| MOVD $0x20, R9 | |
| MOVD $0x30, R10 | |
| LXVD2X (XIP)(R0), VXL // load Xi | |
| LXVD2X (HTBL)(R8), VHL // load pre-computed table | |
| MOVD $0x40, R8 | |
| LVSL (R0)(R0), LEMASK | |
| LXVD2X (HTBL)(R9), VH | |
| MOVD $0x50, R9 | |
| VSPLTISB $0x07, T0 | |
| LXVD2X (HTBL)(R10), VHH | |
| MOVD $0x60, R10 | |
| VXOR LEMASK, T0, LEMASK | |
| LXVD2X (HTBL)(R0), VXC2 | |
| VPERM XL, XL, LEMASK, XL | |
| VXOR ZERO, ZERO, ZERO | |
| CMPU LEN, $64 | |
| BGE gcm_ghash_p8_4x | |
| LXVD2X (INP)(R0), VIN | |
| ADD $16, INP, INP | |
| SUBCCC $16, LEN, LEN | |
| VPERM IN, IN, LEMASK, IN | |
| VXOR IN, XL, IN | |
| BEQ short | |
| LXVD2X (HTBL)(R8), VH2L // load H^2 | |
| MOVD $16, R8 | |
| LXVD2X (HTBL)(R9), VH2 | |
| ADD LEN, INP, R9 // end of input | |
| LXVD2X (HTBL)(R10), VH2H | |
| loop_2x: | |
| LXVD2X (INP)(R0), VIN1 | |
| VPERM IN1, IN1, LEMASK, IN1 | |
| SUBC $32, LEN, LEN | |
| VPMSUMD IN, H2L, XL // H^2.lo·Xi.lo | |
| VPMSUMD IN1, HL, XL1 // H.lo·Xi+1.lo | |
| SUBE R11, R11, R11 // borrow?-1:0 | |
| VPMSUMD IN, H2, XM // H^2.hi·Xi.lo+H^2.lo·Xi.hi | |
| VPMSUMD IN1, H, XM1 // H.hi·Xi+1.lo+H.lo·Xi+1.hi | |
| AND LEN, R11, R11 | |
| VPMSUMD IN, H2H, XH // H^2.hi·Xi.hi | |
| VPMSUMD IN1, HH, XH1 // H.hi·Xi+1.hi | |
| ADD R11, INP, INP | |
| VXOR XL, XL1, XL | |
| VXOR XM, XM1, XM | |
| VPMSUMD XL, XC2, T2 // 1st reduction phase | |
| VSLDOI $8, XM, ZERO, T0 | |
| VSLDOI $8, ZERO, XM, T1 | |
| VXOR XH, XH1, XH | |
| VXOR XL, T0, XL | |
| VXOR XH, T1, XH | |
| VSLDOI $8, XL, XL, XL | |
| VXOR XL, T2, XL | |
| LXVD2X (INP)(R8), VIN | |
| ADD $32, INP, INP | |
| VSLDOI $8, XL, XL, T1 // 2nd reduction phase | |
| VPMSUMD XL, XC2, XL | |
| VPERM IN, IN, LEMASK, IN | |
| VXOR T1, XH, T1 | |
| VXOR IN, T1, IN | |
| VXOR IN, XL, IN | |
| CMP R9, INP | |
| BGT loop_2x // done yet? | |
| CMPWU LEN, $0 | |
| BNE even | |
| short: | |
| VPMSUMD IN, HL, XL // H.lo·Xi.lo | |
| VPMSUMD IN, H, XM // H.hi·Xi.lo+H.lo·Xi.hi | |
| VPMSUMD IN, HH, XH // H.hi·Xi.hi | |
| VPMSUMD XL, XC2, T2 // 1st reduction phase | |
| VSLDOI $8, XM, ZERO, T0 | |
| VSLDOI $8, ZERO, XM, T1 | |
| VXOR XL, T0, XL | |
| VXOR XH, T1, XH | |
| VSLDOI $8, XL, XL, XL | |
| VXOR XL, T2, XL | |
| VSLDOI $8, XL, XL, T1 // 2nd reduction phase | |
| VPMSUMD XL, XC2, XL | |
| VXOR T1, XH, T1 | |
| even: | |
| VXOR XL, T1, XL | |
| VPERM XL, XL, LEMASK, XL | |
| STXVD2X VXL, (XIP+R0) | |
| OR R12, R12, R12 // write out Xi | |
| RET | |
| gcm_ghash_p8_4x: | |
| LVSL (R8)(R0), T0 // 0x0001..0e0f | |
| MOVD $0x70, R8 | |
| LXVD2X (HTBL)(R9), VH2 | |
| MOVD $0x80, R9 | |
| VSPLTISB $8, T1 // 0x0808..0808 | |
| MOVD $0x90, R10 | |
| LXVD2X (HTBL)(R8), VH3L // load H^3 | |
| MOVD $0xa0, R8 | |
| LXVD2X (HTBL)(R9), VH3 | |
| MOVD $0xb0, R9 | |
| LXVD2X (HTBL)(R10), VH3H | |
| MOVD $0xc0, R10 | |
| LXVD2X (HTBL)(R8), VH4L // load H^4 | |
| MOVD $0x10, R8 | |
| LXVD2X (HTBL)(R9), VH4 | |
| MOVD $0x20, R9 | |
| LXVD2X (HTBL)(R10), VH4H | |
| MOVD $0x30, R10 | |
| VSLDOI $8, ZERO, T1, T2 // 0x0000..0808 | |
| VADDUBM T0, T2, HIPERM // 0x0001..1617 | |
| VADDUBM T1, HIPERM, LOPERM // 0x0809..1e1f | |
| SRD $4, LEN, LEN // this allows to use sign bit as carry | |
| LXVD2X (INP)(R0), VIN0 // load input | |
| LXVD2X (INP)(R8), VIN1 | |
| SUBCCC $8, LEN, LEN | |
| LXVD2X (INP)(R9), VIN2 | |
| LXVD2X (INP)(R10), VIN3 | |
| ADD $0x40, INP, INP | |
| VPERM IN0, IN0, LEMASK, IN0 | |
| VPERM IN1, IN1, LEMASK, IN1 | |
| VPERM IN2, IN2, LEMASK, IN2 | |
| VPERM IN3, IN3, LEMASK, IN3 | |
| VXOR IN0, XL, XH | |
| VPMSUMD IN1, H3L, XL1 | |
| VPMSUMD IN1, H3, XM1 | |
| VPMSUMD IN1, H3H, XH1 | |
| VPERM H2, H, HIPERM, H21L | |
| VPERM IN2, IN3, LOPERM, T0 | |
| VPERM H2, H, LOPERM, H21H | |
| VPERM IN2, IN3, HIPERM, T1 | |
| VPMSUMD IN2, H2, XM2 // H^2.lo·Xi+2.hi+H^2.hi·Xi+2.lo | |
| VPMSUMD T0, H21L, XL3 // H^2.lo·Xi+2.lo+H.lo·Xi+3.lo | |
| VPMSUMD IN3, H, XM3 // H.hi·Xi+3.lo +H.lo·Xi+3.hi | |
| VPMSUMD T1, H21H, XH3 // H^2.hi·Xi+2.hi+H.hi·Xi+3.hi | |
| VXOR XM2, XM1, XM2 | |
| VXOR XL3, XL1, XL3 | |
| VXOR XM3, XM2, XM3 | |
| VXOR XH3, XH1, XH3 | |
| BLT tail_4x | |
| loop_4x: | |
| LXVD2X (INP)(R0), VIN0 | |
| LXVD2X (INP)(R8), VIN1 | |
| SUBCCC $4, LEN, LEN | |
| LXVD2X (INP)(R9), VIN2 | |
| LXVD2X (INP)(R10), VIN3 | |
| ADD $0x40, INP, INP | |
| VPERM IN1, IN1, LEMASK, IN1 | |
| VPERM IN2, IN2, LEMASK, IN2 | |
| VPERM IN3, IN3, LEMASK, IN3 | |
| VPERM IN0, IN0, LEMASK, IN0 | |
| VPMSUMD XH, H4L, XL // H^4.lo·Xi.lo | |
| VPMSUMD XH, H4, XM // H^4.hi·Xi.lo+H^4.lo·Xi.hi | |
| VPMSUMD XH, H4H, XH // H^4.hi·Xi.hi | |
| VPMSUMD IN1, H3L, XL1 | |
| VPMSUMD IN1, H3, XM1 | |
| VPMSUMD IN1, H3H, XH1 | |
| VXOR XL, XL3, XL | |
| VXOR XM, XM3, XM | |
| VXOR XH, XH3, XH | |
| VPERM IN2, IN3, LOPERM, T0 | |
| VPERM IN2, IN3, HIPERM, T1 | |
| VPMSUMD XL, XC2, T2 // 1st reduction phase | |
| VPMSUMD T0, H21L, XL3 // H.lo·Xi+3.lo +H^2.lo·Xi+2.lo | |
| VPMSUMD T1, H21H, XH3 // H.hi·Xi+3.hi +H^2.hi·Xi+2.hi | |
| VSLDOI $8, XM, ZERO, T0 | |
| VSLDOI $8, ZERO, XM, T1 | |
| VXOR XL, T0, XL | |
| VXOR XH, T1, XH | |
| VSLDOI $8, XL, XL, XL | |
| VXOR XL, T2, XL | |
| VSLDOI $8, XL, XL, T1 // 2nd reduction phase | |
| VPMSUMD IN2, H2, XM2 // H^2.hi·Xi+2.lo+H^2.lo·Xi+2.hi | |
| VPMSUMD IN3, H, XM3 // H.hi·Xi+3.lo +H.lo·Xi+3.hi | |
| VPMSUMD XL, XC2, XL | |
| VXOR XL3, XL1, XL3 | |
| VXOR XH3, XH1, XH3 | |
| VXOR XH, IN0, XH | |
| VXOR XM2, XM1, XM2 | |
| VXOR XH, T1, XH | |
| VXOR XM3, XM2, XM3 | |
| VXOR XH, XL, XH | |
| BGE loop_4x | |
| tail_4x: | |
| VPMSUMD XH, H4L, XL // H^4.lo·Xi.lo | |
| VPMSUMD XH, H4, XM // H^4.hi·Xi.lo+H^4.lo·Xi.hi | |
| VPMSUMD XH, H4H, XH // H^4.hi·Xi.hi | |
| VXOR XL, XL3, XL | |
| VXOR XM, XM3, XM | |
| VPMSUMD XL, XC2, T2 // 1st reduction phase | |
| VSLDOI $8, XM, ZERO, T0 | |
| VSLDOI $8, ZERO, XM, T1 | |
| VXOR XH, XH3, XH | |
| VXOR XL, T0, XL | |
| VXOR XH, T1, XH | |
| VSLDOI $8, XL, XL, XL | |
| VXOR XL, T2, XL | |
| VSLDOI $8, XL, XL, T1 // 2nd reduction phase | |
| VPMSUMD XL, XC2, XL | |
| VXOR T1, XH, T1 | |
| VXOR XL, T1, XL | |
| ADDCCC $4, LEN, LEN | |
| BEQ done_4x | |
| LXVD2X (INP)(R0), VIN0 | |
| CMPU LEN, $2 | |
| MOVD $-4, LEN | |
| BLT one | |
| LXVD2X (INP)(R8), VIN1 | |
| BEQ two | |
| three: | |
| LXVD2X (INP)(R9), VIN2 | |
| VPERM IN0, IN0, LEMASK, IN0 | |
| VPERM IN1, IN1, LEMASK, IN1 | |
| VPERM IN2, IN2, LEMASK, IN2 | |
| VXOR IN0, XL, XH | |
| VOR H3L, H3L, H4L | |
| VOR H3, H3, H4 | |
| VOR H3H, H3H, H4H | |
| VPERM IN1, IN2, LOPERM, T0 | |
| VPERM IN1, IN2, HIPERM, T1 | |
| VPMSUMD IN1, H2, XM2 // H^2.lo·Xi+1.hi+H^2.hi·Xi+1.lo | |
| VPMSUMD IN2, H, XM3 // H.hi·Xi+2.lo +H.lo·Xi+2.hi | |
| VPMSUMD T0, H21L, XL3 // H^2.lo·Xi+1.lo+H.lo·Xi+2.lo | |
| VPMSUMD T1, H21H, XH3 // H^2.hi·Xi+1.hi+H.hi·Xi+2.hi | |
| VXOR XM3, XM2, XM3 | |
| JMP tail_4x | |
| two: | |
| VPERM IN0, IN0, LEMASK, IN0 | |
| VPERM IN1, IN1, LEMASK, IN1 | |
| VXOR IN, XL, XH | |
| VPERM ZERO, IN1, LOPERM, T0 | |
| VPERM ZERO, IN1, HIPERM, T1 | |
| VSLDOI $8, ZERO, H2, H4L | |
| VOR H2, H2, H4 | |
| VSLDOI $8, H2, ZERO, H4H | |
| VPMSUMD T0, H21L, XL3 // H.lo·Xi+1.lo | |
| VPMSUMD IN1, H, XM3 // H.hi·Xi+1.lo+H.lo·Xi+2.hi | |
| VPMSUMD T1, H21H, XH3 // H.hi·Xi+1.hi | |
| JMP tail_4x | |
| one: | |
| VPERM IN0, IN0, LEMASK, IN0 | |
| VSLDOI $8, ZERO, H, H4L | |
| VOR H, H, H4 | |
| VSLDOI $8, H, ZERO, H4H | |
| VXOR IN0, XL, XH | |
| VXOR XL3, XL3, XL3 | |
| VXOR XM3, XM3, XM3 | |
| VXOR XH3, XH3, XH3 | |
| JMP tail_4x | |
| done_4x: | |
| VPERM XL, XL, LEMASK, XL | |
| STXVD2X VXL, (XIP+R0) // write out Xi | |
| RET | |
| // func gcmMul(output []byte, productTable *[256]byte) | |
| TEXT ·gcmMul(SB), NOSPLIT, $0-32 | |
| MOVD output+0(FP), XIP | |
| MOVD productTable+24(FP), HTBL | |
| MOVD $0x10, R8 | |
| MOVD $0x20, R9 | |
| MOVD $0x30, R10 | |
| LXVD2X (XIP)(R0), VIN // load Xi | |
| LXVD2X (HTBL)(R8), VHL // Load pre-computed table | |
| LVSL (R0)(R0), LEMASK | |
| LXVD2X (HTBL)(R9), VH | |
| VSPLTISB $0x07, T0 | |
| LXVD2X (HTBL)(R10), VHH | |
| VXOR LEMASK, T0, LEMASK | |
| LXVD2X (HTBL)(R0), VXC2 | |
| VPERM IN, IN, LEMASK, IN | |
| VXOR ZERO, ZERO, ZERO | |
| VPMSUMD IN, HL, XL // H.lo·Xi.lo | |
| VPMSUMD IN, H, XM // H.hi·Xi.lo+H.lo·Xi.hi | |
| VPMSUMD IN, HH, XH // H.hi·Xi.hi | |
| VPMSUMD XL, XC2, T2 // 1st reduction phase | |
| VSLDOI $8, XM, ZERO, T0 | |
| VSLDOI $8, ZERO, XM, T1 | |
| VXOR XL, T0, XL | |
| VXOR XH, T1, XH | |
| VSLDOI $8, XL, XL, XL | |
| VXOR XL, T2, XL | |
| VSLDOI $8, XL, XL, T1 // 2nd reduction phase | |
| VPMSUMD XL, XC2, XL | |
| VXOR T1, XH, T1 | |
| VXOR XL, T1, XL | |
| VPERM XL, XL, LEMASK, XL | |
| STXVD2X VXL, (XIP+R0) // write out Xi | |
| RET |