| // Copyright 2015 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // This file contains constant-time, 64-bit assembly implementation of |
| // P256. The optimizations performed here are described in detail in: |
| // S.Gueron and V.Krasnov, "Fast prime field elliptic-curve cryptography with |
| // 256-bit primes" |
| // http://link.springer.com/article/10.1007%2Fs13389-014-0090-x |
| // https://eprint.iacr.org/2013/816.pdf |
| |
| #include "textflag.h" |
| |
| #define res_ptr DI |
| #define x_ptr SI |
| #define y_ptr CX |
| |
| #define acc0 R8 |
| #define acc1 R9 |
| #define acc2 R10 |
| #define acc3 R11 |
| #define acc4 R12 |
| #define acc5 R13 |
| #define t0 R14 |
| #define t1 R15 |
| |
| DATA p256const0<>+0x00(SB)/8, $0x00000000ffffffff |
| DATA p256const1<>+0x00(SB)/8, $0xffffffff00000001 |
| DATA p256ordK0<>+0x00(SB)/8, $0xccd1c8aaee00bc4f |
| DATA p256ord<>+0x00(SB)/8, $0xf3b9cac2fc632551 |
| DATA p256ord<>+0x08(SB)/8, $0xbce6faada7179e84 |
| DATA p256ord<>+0x10(SB)/8, $0xffffffffffffffff |
| DATA p256ord<>+0x18(SB)/8, $0xffffffff00000000 |
| DATA p256one<>+0x00(SB)/8, $0x0000000000000001 |
| DATA p256one<>+0x08(SB)/8, $0xffffffff00000000 |
| DATA p256one<>+0x10(SB)/8, $0xffffffffffffffff |
| DATA p256one<>+0x18(SB)/8, $0x00000000fffffffe |
| GLOBL p256const0<>(SB), 8, $8 |
| GLOBL p256const1<>(SB), 8, $8 |
| GLOBL p256ordK0<>(SB), 8, $8 |
| GLOBL p256ord<>(SB), 8, $32 |
| GLOBL p256one<>(SB), 8, $32 |
| |
| /* ---------------------------------------*/ |
| // func p256LittleToBig(res []byte, in []uint64) |
| TEXT ·p256LittleToBig(SB),NOSPLIT,$0 |
| JMP ·p256BigToLittle(SB) |
| /* ---------------------------------------*/ |
| // func p256BigToLittle(res []uint64, in []byte) |
| TEXT ·p256BigToLittle(SB),NOSPLIT,$0 |
| MOVQ res+0(FP), res_ptr |
| MOVQ in+24(FP), x_ptr |
| |
| MOVQ (8*0)(x_ptr), acc0 |
| MOVQ (8*1)(x_ptr), acc1 |
| MOVQ (8*2)(x_ptr), acc2 |
| MOVQ (8*3)(x_ptr), acc3 |
| |
| BSWAPQ acc0 |
| BSWAPQ acc1 |
| BSWAPQ acc2 |
| BSWAPQ acc3 |
| |
| MOVQ acc3, (8*0)(res_ptr) |
| MOVQ acc2, (8*1)(res_ptr) |
| MOVQ acc1, (8*2)(res_ptr) |
| MOVQ acc0, (8*3)(res_ptr) |
| |
| RET |
| /* ---------------------------------------*/ |
| // func p256MovCond(res, a, b []uint64, cond int) |
| // If cond == 0 res=b, else res=a |
| TEXT ·p256MovCond(SB),NOSPLIT,$0 |
| MOVQ res+0(FP), res_ptr |
| MOVQ a+24(FP), x_ptr |
| MOVQ b+48(FP), y_ptr |
| MOVQ cond+72(FP), X12 |
| |
| PXOR X13, X13 |
| PSHUFD $0, X12, X12 |
| PCMPEQL X13, X12 |
| |
| MOVOU X12, X0 |
| PANDN (16*0)(x_ptr), X0 |
| MOVOU X12, X1 |
| PANDN (16*1)(x_ptr), X1 |
| MOVOU X12, X2 |
| PANDN (16*2)(x_ptr), X2 |
| MOVOU X12, X3 |
| PANDN (16*3)(x_ptr), X3 |
| MOVOU X12, X4 |
| PANDN (16*4)(x_ptr), X4 |
| MOVOU X12, X5 |
| PANDN (16*5)(x_ptr), X5 |
| |
| MOVOU (16*0)(y_ptr), X6 |
| MOVOU (16*1)(y_ptr), X7 |
| MOVOU (16*2)(y_ptr), X8 |
| MOVOU (16*3)(y_ptr), X9 |
| MOVOU (16*4)(y_ptr), X10 |
| MOVOU (16*5)(y_ptr), X11 |
| |
| PAND X12, X6 |
| PAND X12, X7 |
| PAND X12, X8 |
| PAND X12, X9 |
| PAND X12, X10 |
| PAND X12, X11 |
| |
| PXOR X6, X0 |
| PXOR X7, X1 |
| PXOR X8, X2 |
| PXOR X9, X3 |
| PXOR X10, X4 |
| PXOR X11, X5 |
| |
| MOVOU X0, (16*0)(res_ptr) |
| MOVOU X1, (16*1)(res_ptr) |
| MOVOU X2, (16*2)(res_ptr) |
| MOVOU X3, (16*3)(res_ptr) |
| MOVOU X4, (16*4)(res_ptr) |
| MOVOU X5, (16*5)(res_ptr) |
| |
| RET |
| /* ---------------------------------------*/ |
| // func p256NegCond(val []uint64, cond int) |
| TEXT ·p256NegCond(SB),NOSPLIT,$0 |
| MOVQ val+0(FP), res_ptr |
| MOVQ cond+24(FP), t0 |
| // acc = poly |
| MOVQ $-1, acc0 |
| MOVQ p256const0<>(SB), acc1 |
| MOVQ $0, acc2 |
| MOVQ p256const1<>(SB), acc3 |
| // Load the original value |
| MOVQ (8*0)(res_ptr), acc5 |
| MOVQ (8*1)(res_ptr), x_ptr |
| MOVQ (8*2)(res_ptr), y_ptr |
| MOVQ (8*3)(res_ptr), t1 |
| // Speculatively subtract |
| SUBQ acc5, acc0 |
| SBBQ x_ptr, acc1 |
| SBBQ y_ptr, acc2 |
| SBBQ t1, acc3 |
| // If condition is 0, keep original value |
| TESTQ t0, t0 |
| CMOVQEQ acc5, acc0 |
| CMOVQEQ x_ptr, acc1 |
| CMOVQEQ y_ptr, acc2 |
| CMOVQEQ t1, acc3 |
| // Store result |
| MOVQ acc0, (8*0)(res_ptr) |
| MOVQ acc1, (8*1)(res_ptr) |
| MOVQ acc2, (8*2)(res_ptr) |
| MOVQ acc3, (8*3)(res_ptr) |
| |
| RET |
| /* ---------------------------------------*/ |
| // func p256Sqr(res, in []uint64) |
| TEXT ·p256Sqr(SB),NOSPLIT,$0 |
| MOVQ res+0(FP), res_ptr |
| MOVQ in+24(FP), x_ptr |
| // y[1:] * y[0] |
| MOVQ (8*0)(x_ptr), t0 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| MOVQ AX, acc1 |
| MOVQ DX, acc2 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, acc3 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, acc4 |
| // y[2:] * y[1] |
| MOVQ (8*1)(x_ptr), t0 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, acc5 |
| // y[3] * y[2] |
| MOVQ (8*2)(x_ptr), t0 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc5 |
| ADCQ $0, DX |
| MOVQ DX, y_ptr |
| XORQ t1, t1 |
| // *2 |
| ADDQ acc1, acc1 |
| ADCQ acc2, acc2 |
| ADCQ acc3, acc3 |
| ADCQ acc4, acc4 |
| ADCQ acc5, acc5 |
| ADCQ y_ptr, y_ptr |
| ADCQ $0, t1 |
| // Missing products |
| MOVQ (8*0)(x_ptr), AX |
| MULQ AX |
| MOVQ AX, acc0 |
| MOVQ DX, t0 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ AX |
| ADDQ t0, acc1 |
| ADCQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t0 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ AX |
| ADDQ t0, acc3 |
| ADCQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, t0 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ AX |
| ADDQ t0, acc5 |
| ADCQ AX, y_ptr |
| ADCQ DX, t1 |
| MOVQ t1, x_ptr |
| // First reduction step |
| MOVQ acc0, AX |
| MOVQ acc0, t1 |
| SHLQ $32, acc0 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc0, acc1 |
| ADCQ t1, acc2 |
| ADCQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, acc0 |
| // Second reduction step |
| MOVQ acc1, AX |
| MOVQ acc1, t1 |
| SHLQ $32, acc1 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc1, acc2 |
| ADCQ t1, acc3 |
| ADCQ AX, acc0 |
| ADCQ $0, DX |
| MOVQ DX, acc1 |
| // Third reduction step |
| MOVQ acc2, AX |
| MOVQ acc2, t1 |
| SHLQ $32, acc2 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc2, acc3 |
| ADCQ t1, acc0 |
| ADCQ AX, acc1 |
| ADCQ $0, DX |
| MOVQ DX, acc2 |
| // Last reduction step |
| XORQ t0, t0 |
| MOVQ acc3, AX |
| MOVQ acc3, t1 |
| SHLQ $32, acc3 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc3, acc0 |
| ADCQ t1, acc1 |
| ADCQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, acc3 |
| // Add bits [511:256] of the sqr result |
| ADCQ acc4, acc0 |
| ADCQ acc5, acc1 |
| ADCQ y_ptr, acc2 |
| ADCQ x_ptr, acc3 |
| ADCQ $0, t0 |
| |
| MOVQ acc0, acc4 |
| MOVQ acc1, acc5 |
| MOVQ acc2, y_ptr |
| MOVQ acc3, t1 |
| // Subtract p256 |
| SUBQ $-1, acc0 |
| SBBQ p256const0<>(SB) ,acc1 |
| SBBQ $0, acc2 |
| SBBQ p256const1<>(SB), acc3 |
| SBBQ $0, t0 |
| |
| CMOVQCS acc4, acc0 |
| CMOVQCS acc5, acc1 |
| CMOVQCS y_ptr, acc2 |
| CMOVQCS t1, acc3 |
| |
| MOVQ acc0, (8*0)(res_ptr) |
| MOVQ acc1, (8*1)(res_ptr) |
| MOVQ acc2, (8*2)(res_ptr) |
| MOVQ acc3, (8*3)(res_ptr) |
| |
| RET |
| /* ---------------------------------------*/ |
| // func p256Mul(res, in1, in2 []uint64) |
| TEXT ·p256Mul(SB),NOSPLIT,$0 |
| MOVQ res+0(FP), res_ptr |
| MOVQ in1+24(FP), x_ptr |
| MOVQ in2+48(FP), y_ptr |
| // x * y[0] |
| MOVQ (8*0)(y_ptr), t0 |
| |
| MOVQ (8*0)(x_ptr), AX |
| MULQ t0 |
| MOVQ AX, acc0 |
| MOVQ DX, acc1 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc1 |
| ADCQ $0, DX |
| MOVQ DX, acc2 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, acc3 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, acc4 |
| XORQ acc5, acc5 |
| // First reduction step |
| MOVQ acc0, AX |
| MOVQ acc0, t1 |
| SHLQ $32, acc0 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc0, acc1 |
| ADCQ t1, acc2 |
| ADCQ AX, acc3 |
| ADCQ DX, acc4 |
| ADCQ $0, acc5 |
| XORQ acc0, acc0 |
| // x * y[1] |
| MOVQ (8*1)(y_ptr), t0 |
| |
| MOVQ (8*0)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc1 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc2 |
| ADCQ $0, DX |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc3 |
| ADCQ $0, DX |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ DX, acc5 |
| ADCQ $0, acc0 |
| // Second reduction step |
| MOVQ acc1, AX |
| MOVQ acc1, t1 |
| SHLQ $32, acc1 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc1, acc2 |
| ADCQ t1, acc3 |
| ADCQ AX, acc4 |
| ADCQ DX, acc5 |
| ADCQ $0, acc0 |
| XORQ acc1, acc1 |
| // x * y[2] |
| MOVQ (8*2)(y_ptr), t0 |
| |
| MOVQ (8*0)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc3 |
| ADCQ $0, DX |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc5 |
| ADCQ $0, DX |
| ADDQ AX, acc5 |
| ADCQ DX, acc0 |
| ADCQ $0, acc1 |
| // Third reduction step |
| MOVQ acc2, AX |
| MOVQ acc2, t1 |
| SHLQ $32, acc2 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc2, acc3 |
| ADCQ t1, acc4 |
| ADCQ AX, acc5 |
| ADCQ DX, acc0 |
| ADCQ $0, acc1 |
| XORQ acc2, acc2 |
| // x * y[3] |
| MOVQ (8*3)(y_ptr), t0 |
| |
| MOVQ (8*0)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc5 |
| ADCQ $0, DX |
| ADDQ AX, acc5 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc0 |
| ADCQ $0, DX |
| ADDQ AX, acc0 |
| ADCQ DX, acc1 |
| ADCQ $0, acc2 |
| // Last reduction step |
| MOVQ acc3, AX |
| MOVQ acc3, t1 |
| SHLQ $32, acc3 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc3, acc4 |
| ADCQ t1, acc5 |
| ADCQ AX, acc0 |
| ADCQ DX, acc1 |
| ADCQ $0, acc2 |
| // Copy result [255:0] |
| MOVQ acc4, x_ptr |
| MOVQ acc5, acc3 |
| MOVQ acc0, t0 |
| MOVQ acc1, t1 |
| // Subtract p256 |
| SUBQ $-1, acc4 |
| SBBQ p256const0<>(SB) ,acc5 |
| SBBQ $0, acc0 |
| SBBQ p256const1<>(SB), acc1 |
| SBBQ $0, acc2 |
| |
| CMOVQCS x_ptr, acc4 |
| CMOVQCS acc3, acc5 |
| CMOVQCS t0, acc0 |
| CMOVQCS t1, acc1 |
| |
| MOVQ acc4, (8*0)(res_ptr) |
| MOVQ acc5, (8*1)(res_ptr) |
| MOVQ acc0, (8*2)(res_ptr) |
| MOVQ acc1, (8*3)(res_ptr) |
| |
| RET |
| /* ---------------------------------------*/ |
| // func p256FromMont(res, in []uint64) |
| TEXT ·p256FromMont(SB),NOSPLIT,$0 |
| MOVQ res+0(FP), res_ptr |
| MOVQ in+24(FP), x_ptr |
| |
| MOVQ (8*0)(x_ptr), acc0 |
| MOVQ (8*1)(x_ptr), acc1 |
| MOVQ (8*2)(x_ptr), acc2 |
| MOVQ (8*3)(x_ptr), acc3 |
| XORQ acc4, acc4 |
| |
| // Only reduce, no multiplications are needed |
| // First stage |
| MOVQ acc0, AX |
| MOVQ acc0, t1 |
| SHLQ $32, acc0 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc0, acc1 |
| ADCQ t1, acc2 |
| ADCQ AX, acc3 |
| ADCQ DX, acc4 |
| XORQ acc5, acc5 |
| // Second stage |
| MOVQ acc1, AX |
| MOVQ acc1, t1 |
| SHLQ $32, acc1 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc1, acc2 |
| ADCQ t1, acc3 |
| ADCQ AX, acc4 |
| ADCQ DX, acc5 |
| XORQ acc0, acc0 |
| // Third stage |
| MOVQ acc2, AX |
| MOVQ acc2, t1 |
| SHLQ $32, acc2 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc2, acc3 |
| ADCQ t1, acc4 |
| ADCQ AX, acc5 |
| ADCQ DX, acc0 |
| XORQ acc1, acc1 |
| // Last stage |
| MOVQ acc3, AX |
| MOVQ acc3, t1 |
| SHLQ $32, acc3 |
| MULQ p256const1<>(SB) |
| SHRQ $32, t1 |
| ADDQ acc3, acc4 |
| ADCQ t1, acc5 |
| ADCQ AX, acc0 |
| ADCQ DX, acc1 |
| |
| MOVQ acc4, x_ptr |
| MOVQ acc5, acc3 |
| MOVQ acc0, t0 |
| MOVQ acc1, t1 |
| |
| SUBQ $-1, acc4 |
| SBBQ p256const0<>(SB), acc5 |
| SBBQ $0, acc0 |
| SBBQ p256const1<>(SB), acc1 |
| |
| CMOVQCS x_ptr, acc4 |
| CMOVQCS acc3, acc5 |
| CMOVQCS t0, acc0 |
| CMOVQCS t1, acc1 |
| |
| MOVQ acc4, (8*0)(res_ptr) |
| MOVQ acc5, (8*1)(res_ptr) |
| MOVQ acc0, (8*2)(res_ptr) |
| MOVQ acc1, (8*3)(res_ptr) |
| |
| RET |
| /* ---------------------------------------*/ |
| // Constant time point access to arbitrary point table. |
| // Indexed from 1 to 15, with -1 offset |
| // (index 0 is implicitly point at infinity) |
| // func p256Select(point, table []uint64, idx int) |
| TEXT ·p256Select(SB),NOSPLIT,$0 |
| MOVQ idx+48(FP),AX |
| MOVQ table+24(FP),DI |
| MOVQ point+0(FP),DX |
| |
| PXOR X15, X15 // X15 = 0 |
| PCMPEQL X14, X14 // X14 = -1 |
| PSUBL X14, X15 // X15 = 1 |
| MOVL AX, X14 |
| PSHUFD $0, X14, X14 |
| |
| PXOR X0, X0 |
| PXOR X1, X1 |
| PXOR X2, X2 |
| PXOR X3, X3 |
| PXOR X4, X4 |
| PXOR X5, X5 |
| MOVQ $16, AX |
| |
| MOVOU X15, X13 |
| |
| loop_select: |
| |
| MOVOU X13, X12 |
| PADDL X15, X13 |
| PCMPEQL X14, X12 |
| |
| MOVOU (16*0)(DI), X6 |
| MOVOU (16*1)(DI), X7 |
| MOVOU (16*2)(DI), X8 |
| MOVOU (16*3)(DI), X9 |
| MOVOU (16*4)(DI), X10 |
| MOVOU (16*5)(DI), X11 |
| ADDQ $(16*6), DI |
| |
| PAND X12, X6 |
| PAND X12, X7 |
| PAND X12, X8 |
| PAND X12, X9 |
| PAND X12, X10 |
| PAND X12, X11 |
| |
| PXOR X6, X0 |
| PXOR X7, X1 |
| PXOR X8, X2 |
| PXOR X9, X3 |
| PXOR X10, X4 |
| PXOR X11, X5 |
| |
| DECQ AX |
| JNE loop_select |
| |
| MOVOU X0, (16*0)(DX) |
| MOVOU X1, (16*1)(DX) |
| MOVOU X2, (16*2)(DX) |
| MOVOU X3, (16*3)(DX) |
| MOVOU X4, (16*4)(DX) |
| MOVOU X5, (16*5)(DX) |
| |
| RET |
| /* ---------------------------------------*/ |
| // Constant time point access to base point table. |
| // func p256SelectBase(point, table []uint64, idx int) |
| TEXT ·p256SelectBase(SB),NOSPLIT,$0 |
| MOVQ idx+48(FP),AX |
| MOVQ table+24(FP),DI |
| MOVQ point+0(FP),DX |
| |
| PXOR X15, X15 // X15 = 0 |
| PCMPEQL X14, X14 // X14 = -1 |
| PSUBL X14, X15 // X15 = 1 |
| MOVL AX, X14 |
| PSHUFD $0, X14, X14 |
| |
| PXOR X0, X0 |
| PXOR X1, X1 |
| PXOR X2, X2 |
| PXOR X3, X3 |
| MOVQ $32, AX |
| |
| MOVOU X15, X13 |
| |
| loop_select_base: |
| |
| MOVOU X13, X12 |
| PADDL X15, X13 |
| PCMPEQL X14, X12 |
| |
| MOVOU (16*0)(DI), X4 |
| MOVOU (16*1)(DI), X5 |
| MOVOU (16*2)(DI), X6 |
| MOVOU (16*3)(DI), X7 |
| |
| MOVOU (16*4)(DI), X8 |
| MOVOU (16*5)(DI), X9 |
| MOVOU (16*6)(DI), X10 |
| MOVOU (16*7)(DI), X11 |
| |
| ADDQ $(16*8), DI |
| |
| PAND X12, X4 |
| PAND X12, X5 |
| PAND X12, X6 |
| PAND X12, X7 |
| |
| MOVOU X13, X12 |
| PADDL X15, X13 |
| PCMPEQL X14, X12 |
| |
| PAND X12, X8 |
| PAND X12, X9 |
| PAND X12, X10 |
| PAND X12, X11 |
| |
| PXOR X4, X0 |
| PXOR X5, X1 |
| PXOR X6, X2 |
| PXOR X7, X3 |
| |
| PXOR X8, X0 |
| PXOR X9, X1 |
| PXOR X10, X2 |
| PXOR X11, X3 |
| |
| DECQ AX |
| JNE loop_select_base |
| |
| MOVOU X0, (16*0)(DX) |
| MOVOU X1, (16*1)(DX) |
| MOVOU X2, (16*2)(DX) |
| MOVOU X3, (16*3)(DX) |
| |
| RET |
| /* ---------------------------------------*/ |
| // func p256OrdMul(res, in1, in2 []uint64) |
| TEXT ·p256OrdMul(SB),NOSPLIT,$0 |
| MOVQ res+0(FP), res_ptr |
| MOVQ in1+24(FP), x_ptr |
| MOVQ in2+48(FP), y_ptr |
| // x * y[0] |
| MOVQ (8*0)(y_ptr), t0 |
| |
| MOVQ (8*0)(x_ptr), AX |
| MULQ t0 |
| MOVQ AX, acc0 |
| MOVQ DX, acc1 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc1 |
| ADCQ $0, DX |
| MOVQ DX, acc2 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, acc3 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, acc4 |
| XORQ acc5, acc5 |
| // First reduction step |
| MOVQ acc0, AX |
| MULQ p256ordK0<>(SB) |
| MOVQ AX, t0 |
| |
| MOVQ p256ord<>+0x00(SB), AX |
| MULQ t0 |
| ADDQ AX, acc0 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x08(SB), AX |
| MULQ t0 |
| ADDQ t1, acc1 |
| ADCQ $0, DX |
| ADDQ AX, acc1 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x10(SB), AX |
| MULQ t0 |
| ADDQ t1, acc2 |
| ADCQ $0, DX |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x18(SB), AX |
| MULQ t0 |
| ADDQ t1, acc3 |
| ADCQ $0, DX |
| ADDQ AX, acc3 |
| ADCQ DX, acc4 |
| ADCQ $0, acc5 |
| // x * y[1] |
| MOVQ (8*1)(y_ptr), t0 |
| |
| MOVQ (8*0)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc1 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc2 |
| ADCQ $0, DX |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc3 |
| ADCQ $0, DX |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ DX, acc5 |
| ADCQ $0, acc0 |
| // Second reduction step |
| MOVQ acc1, AX |
| MULQ p256ordK0<>(SB) |
| MOVQ AX, t0 |
| |
| MOVQ p256ord<>+0x00(SB), AX |
| MULQ t0 |
| ADDQ AX, acc1 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x08(SB), AX |
| MULQ t0 |
| ADDQ t1, acc2 |
| ADCQ $0, DX |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x10(SB), AX |
| MULQ t0 |
| ADDQ t1, acc3 |
| ADCQ $0, DX |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x18(SB), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ DX, acc5 |
| ADCQ $0, acc0 |
| // x * y[2] |
| MOVQ (8*2)(y_ptr), t0 |
| |
| MOVQ (8*0)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc3 |
| ADCQ $0, DX |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc5 |
| ADCQ $0, DX |
| ADDQ AX, acc5 |
| ADCQ DX, acc0 |
| ADCQ $0, acc1 |
| // Third reduction step |
| MOVQ acc2, AX |
| MULQ p256ordK0<>(SB) |
| MOVQ AX, t0 |
| |
| MOVQ p256ord<>+0x00(SB), AX |
| MULQ t0 |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x08(SB), AX |
| MULQ t0 |
| ADDQ t1, acc3 |
| ADCQ $0, DX |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x10(SB), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x18(SB), AX |
| MULQ t0 |
| ADDQ t1, acc5 |
| ADCQ $0, DX |
| ADDQ AX, acc5 |
| ADCQ DX, acc0 |
| ADCQ $0, acc1 |
| // x * y[3] |
| MOVQ (8*3)(y_ptr), t0 |
| |
| MOVQ (8*0)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc5 |
| ADCQ $0, DX |
| ADDQ AX, acc5 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc0 |
| ADCQ $0, DX |
| ADDQ AX, acc0 |
| ADCQ DX, acc1 |
| ADCQ $0, acc2 |
| // Last reduction step |
| MOVQ acc3, AX |
| MULQ p256ordK0<>(SB) |
| MOVQ AX, t0 |
| |
| MOVQ p256ord<>+0x00(SB), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x08(SB), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x10(SB), AX |
| MULQ t0 |
| ADDQ t1, acc5 |
| ADCQ $0, DX |
| ADDQ AX, acc5 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x18(SB), AX |
| MULQ t0 |
| ADDQ t1, acc0 |
| ADCQ $0, DX |
| ADDQ AX, acc0 |
| ADCQ DX, acc1 |
| ADCQ $0, acc2 |
| // Copy result [255:0] |
| MOVQ acc4, x_ptr |
| MOVQ acc5, acc3 |
| MOVQ acc0, t0 |
| MOVQ acc1, t1 |
| // Subtract p256 |
| SUBQ p256ord<>+0x00(SB), acc4 |
| SBBQ p256ord<>+0x08(SB) ,acc5 |
| SBBQ p256ord<>+0x10(SB), acc0 |
| SBBQ p256ord<>+0x18(SB), acc1 |
| SBBQ $0, acc2 |
| |
| CMOVQCS x_ptr, acc4 |
| CMOVQCS acc3, acc5 |
| CMOVQCS t0, acc0 |
| CMOVQCS t1, acc1 |
| |
| MOVQ acc4, (8*0)(res_ptr) |
| MOVQ acc5, (8*1)(res_ptr) |
| MOVQ acc0, (8*2)(res_ptr) |
| MOVQ acc1, (8*3)(res_ptr) |
| |
| RET |
| /* ---------------------------------------*/ |
| // func p256OrdSqr(res, in []uint64, n int) |
| TEXT ·p256OrdSqr(SB),NOSPLIT,$0 |
| MOVQ res+0(FP), res_ptr |
| MOVQ in+24(FP), x_ptr |
| MOVQ n+48(FP), BX |
| |
| ordSqrLoop: |
| |
| // y[1:] * y[0] |
| MOVQ (8*0)(x_ptr), t0 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ t0 |
| MOVQ AX, acc1 |
| MOVQ DX, acc2 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, acc3 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, acc4 |
| // y[2:] * y[1] |
| MOVQ (8*1)(x_ptr), t0 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ t1, acc4 |
| ADCQ $0, DX |
| ADDQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, acc5 |
| // y[3] * y[2] |
| MOVQ (8*2)(x_ptr), t0 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ t0 |
| ADDQ AX, acc5 |
| ADCQ $0, DX |
| MOVQ DX, y_ptr |
| XORQ t1, t1 |
| // *2 |
| ADDQ acc1, acc1 |
| ADCQ acc2, acc2 |
| ADCQ acc3, acc3 |
| ADCQ acc4, acc4 |
| ADCQ acc5, acc5 |
| ADCQ y_ptr, y_ptr |
| ADCQ $0, t1 |
| // Missing products |
| MOVQ (8*0)(x_ptr), AX |
| MULQ AX |
| MOVQ AX, acc0 |
| MOVQ DX, t0 |
| |
| MOVQ (8*1)(x_ptr), AX |
| MULQ AX |
| ADDQ t0, acc1 |
| ADCQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t0 |
| |
| MOVQ (8*2)(x_ptr), AX |
| MULQ AX |
| ADDQ t0, acc3 |
| ADCQ AX, acc4 |
| ADCQ $0, DX |
| MOVQ DX, t0 |
| |
| MOVQ (8*3)(x_ptr), AX |
| MULQ AX |
| ADDQ t0, acc5 |
| ADCQ AX, y_ptr |
| ADCQ DX, t1 |
| MOVQ t1, x_ptr |
| // First reduction step |
| MOVQ acc0, AX |
| MULQ p256ordK0<>(SB) |
| MOVQ AX, t0 |
| |
| MOVQ p256ord<>+0x00(SB), AX |
| MULQ t0 |
| ADDQ AX, acc0 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x08(SB), AX |
| MULQ t0 |
| ADDQ t1, acc1 |
| ADCQ $0, DX |
| ADDQ AX, acc1 |
| |
| MOVQ t0, t1 |
| ADCQ DX, acc2 |
| ADCQ $0, t1 |
| SUBQ t0, acc2 |
| SBBQ $0, t1 |
| |
| MOVQ t0, AX |
| MOVQ t0, DX |
| MOVQ t0, acc0 |
| SHLQ $32, AX |
| SHRQ $32, DX |
| |
| ADDQ t1, acc3 |
| ADCQ $0, acc0 |
| SUBQ AX, acc3 |
| SBBQ DX, acc0 |
| // Second reduction step |
| MOVQ acc1, AX |
| MULQ p256ordK0<>(SB) |
| MOVQ AX, t0 |
| |
| MOVQ p256ord<>+0x00(SB), AX |
| MULQ t0 |
| ADDQ AX, acc1 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x08(SB), AX |
| MULQ t0 |
| ADDQ t1, acc2 |
| ADCQ $0, DX |
| ADDQ AX, acc2 |
| |
| MOVQ t0, t1 |
| ADCQ DX, acc3 |
| ADCQ $0, t1 |
| SUBQ t0, acc3 |
| SBBQ $0, t1 |
| |
| MOVQ t0, AX |
| MOVQ t0, DX |
| MOVQ t0, acc1 |
| SHLQ $32, AX |
| SHRQ $32, DX |
| |
| ADDQ t1, acc0 |
| ADCQ $0, acc1 |
| SUBQ AX, acc0 |
| SBBQ DX, acc1 |
| // Third reduction step |
| MOVQ acc2, AX |
| MULQ p256ordK0<>(SB) |
| MOVQ AX, t0 |
| |
| MOVQ p256ord<>+0x00(SB), AX |
| MULQ t0 |
| ADDQ AX, acc2 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x08(SB), AX |
| MULQ t0 |
| ADDQ t1, acc3 |
| ADCQ $0, DX |
| ADDQ AX, acc3 |
| |
| MOVQ t0, t1 |
| ADCQ DX, acc0 |
| ADCQ $0, t1 |
| SUBQ t0, acc0 |
| SBBQ $0, t1 |
| |
| MOVQ t0, AX |
| MOVQ t0, DX |
| MOVQ t0, acc2 |
| SHLQ $32, AX |
| SHRQ $32, DX |
| |
| ADDQ t1, acc1 |
| ADCQ $0, acc2 |
| SUBQ AX, acc1 |
| SBBQ DX, acc2 |
| // Last reduction step |
| MOVQ acc3, AX |
| MULQ p256ordK0<>(SB) |
| MOVQ AX, t0 |
| |
| MOVQ p256ord<>+0x00(SB), AX |
| MULQ t0 |
| ADDQ AX, acc3 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ p256ord<>+0x08(SB), AX |
| MULQ t0 |
| ADDQ t1, acc0 |
| ADCQ $0, DX |
| ADDQ AX, acc0 |
| ADCQ $0, DX |
| MOVQ DX, t1 |
| |
| MOVQ t0, t1 |
| ADCQ DX, acc1 |
| ADCQ $0, t1 |
| SUBQ t0, acc1 |
| SBBQ $0, t1 |
| |
| MOVQ t0, AX |
| MOVQ t0, DX |
| MOVQ t0, acc3 |
| SHLQ $32, AX |
| SHRQ $32, DX |
| |
| ADDQ t1, acc2 |
| ADCQ $0, acc3 |
| SUBQ AX, acc2 |
| SBBQ DX, acc3 |
| XORQ t0, t0 |
| // Add bits [511:256] of the sqr result |
| ADCQ acc4, acc0 |
| ADCQ acc5, acc1 |
| ADCQ y_ptr, acc2 |
| ADCQ x_ptr, acc3 |
| ADCQ $0, t0 |
| |
| MOVQ acc0, acc4 |
| MOVQ acc1, acc5 |
| MOVQ acc2, y_ptr |
| MOVQ acc3, t1 |
| // Subtract p256 |
| SUBQ p256ord<>+0x00(SB), acc0 |
| SBBQ p256ord<>+0x08(SB) ,acc1 |
| SBBQ p256ord<>+0x10(SB), acc2 |
| SBBQ p256ord<>+0x18(SB), acc3 |
| SBBQ $0, t0 |
| |
| CMOVQCS acc4, acc0 |
| CMOVQCS acc5, acc1 |
| CMOVQCS y_ptr, acc2 |
| CMOVQCS t1, acc3 |
| |
| MOVQ acc0, (8*0)(res_ptr) |
| MOVQ acc1, (8*1)(res_ptr) |
| MOVQ acc2, (8*2)(res_ptr) |
| MOVQ acc3, (8*3)(res_ptr) |
| MOVQ res_ptr, x_ptr |
| DECQ BX |
| JNE ordSqrLoop |
| |
| RET |
| /* ---------------------------------------*/ |
| #undef res_ptr |
| #undef x_ptr |
| #undef y_ptr |
| |
| #undef acc0 |
| #undef acc1 |
| #undef acc2 |
| #undef acc3 |
| #undef acc4 |
| #undef acc5 |
| #undef t0 |
| #undef t1 |
| /* ---------------------------------------*/ |
| #define mul0 AX |
| #define mul1 DX |
| #define acc0 BX |
| #define acc1 CX |
| #define acc2 R8 |
| #define acc3 R9 |
| #define acc4 R10 |
| #define acc5 R11 |
| #define acc6 R12 |
| #define acc7 R13 |
| #define t0 R14 |
| #define t1 R15 |
| #define t2 DI |
| #define t3 SI |
| #define hlp BP |
| /* ---------------------------------------*/ |
| TEXT p256SubInternal(SB),NOSPLIT,$0 |
| XORQ mul0, mul0 |
| SUBQ t0, acc4 |
| SBBQ t1, acc5 |
| SBBQ t2, acc6 |
| SBBQ t3, acc7 |
| SBBQ $0, mul0 |
| |
| MOVQ acc4, acc0 |
| MOVQ acc5, acc1 |
| MOVQ acc6, acc2 |
| MOVQ acc7, acc3 |
| |
| ADDQ $-1, acc4 |
| ADCQ p256const0<>(SB), acc5 |
| ADCQ $0, acc6 |
| ADCQ p256const1<>(SB), acc7 |
| ADCQ $0, mul0 |
| |
| CMOVQNE acc0, acc4 |
| CMOVQNE acc1, acc5 |
| CMOVQNE acc2, acc6 |
| CMOVQNE acc3, acc7 |
| |
| RET |
| /* ---------------------------------------*/ |
| TEXT p256MulInternal(SB),NOSPLIT,$0 |
| MOVQ acc4, mul0 |
| MULQ t0 |
| MOVQ mul0, acc0 |
| MOVQ mul1, acc1 |
| |
| MOVQ acc4, mul0 |
| MULQ t1 |
| ADDQ mul0, acc1 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc2 |
| |
| MOVQ acc4, mul0 |
| MULQ t2 |
| ADDQ mul0, acc2 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc3 |
| |
| MOVQ acc4, mul0 |
| MULQ t3 |
| ADDQ mul0, acc3 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc4 |
| |
| MOVQ acc5, mul0 |
| MULQ t0 |
| ADDQ mul0, acc1 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc5, mul0 |
| MULQ t1 |
| ADDQ hlp, acc2 |
| ADCQ $0, mul1 |
| ADDQ mul0, acc2 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc5, mul0 |
| MULQ t2 |
| ADDQ hlp, acc3 |
| ADCQ $0, mul1 |
| ADDQ mul0, acc3 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc5, mul0 |
| MULQ t3 |
| ADDQ hlp, acc4 |
| ADCQ $0, mul1 |
| ADDQ mul0, acc4 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc5 |
| |
| MOVQ acc6, mul0 |
| MULQ t0 |
| ADDQ mul0, acc2 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc6, mul0 |
| MULQ t1 |
| ADDQ hlp, acc3 |
| ADCQ $0, mul1 |
| ADDQ mul0, acc3 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc6, mul0 |
| MULQ t2 |
| ADDQ hlp, acc4 |
| ADCQ $0, mul1 |
| ADDQ mul0, acc4 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc6, mul0 |
| MULQ t3 |
| ADDQ hlp, acc5 |
| ADCQ $0, mul1 |
| ADDQ mul0, acc5 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc6 |
| |
| MOVQ acc7, mul0 |
| MULQ t0 |
| ADDQ mul0, acc3 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc7, mul0 |
| MULQ t1 |
| ADDQ hlp, acc4 |
| ADCQ $0, mul1 |
| ADDQ mul0, acc4 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc7, mul0 |
| MULQ t2 |
| ADDQ hlp, acc5 |
| ADCQ $0, mul1 |
| ADDQ mul0, acc5 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc7, mul0 |
| MULQ t3 |
| ADDQ hlp, acc6 |
| ADCQ $0, mul1 |
| ADDQ mul0, acc6 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc7 |
| // First reduction step |
| MOVQ acc0, mul0 |
| MOVQ acc0, hlp |
| SHLQ $32, acc0 |
| MULQ p256const1<>(SB) |
| SHRQ $32, hlp |
| ADDQ acc0, acc1 |
| ADCQ hlp, acc2 |
| ADCQ mul0, acc3 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc0 |
| // Second reduction step |
| MOVQ acc1, mul0 |
| MOVQ acc1, hlp |
| SHLQ $32, acc1 |
| MULQ p256const1<>(SB) |
| SHRQ $32, hlp |
| ADDQ acc1, acc2 |
| ADCQ hlp, acc3 |
| ADCQ mul0, acc0 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc1 |
| // Third reduction step |
| MOVQ acc2, mul0 |
| MOVQ acc2, hlp |
| SHLQ $32, acc2 |
| MULQ p256const1<>(SB) |
| SHRQ $32, hlp |
| ADDQ acc2, acc3 |
| ADCQ hlp, acc0 |
| ADCQ mul0, acc1 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc2 |
| // Last reduction step |
| MOVQ acc3, mul0 |
| MOVQ acc3, hlp |
| SHLQ $32, acc3 |
| MULQ p256const1<>(SB) |
| SHRQ $32, hlp |
| ADDQ acc3, acc0 |
| ADCQ hlp, acc1 |
| ADCQ mul0, acc2 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc3 |
| BYTE $0x48; BYTE $0xc7; BYTE $0xc5; BYTE $0x00; BYTE $0x00; BYTE $0x00; BYTE $0x00 // MOVQ $0, BP |
| // Add bits [511:256] of the result |
| ADCQ acc0, acc4 |
| ADCQ acc1, acc5 |
| ADCQ acc2, acc6 |
| ADCQ acc3, acc7 |
| ADCQ $0, hlp |
| // Copy result |
| MOVQ acc4, acc0 |
| MOVQ acc5, acc1 |
| MOVQ acc6, acc2 |
| MOVQ acc7, acc3 |
| // Subtract p256 |
| SUBQ $-1, acc4 |
| SBBQ p256const0<>(SB) ,acc5 |
| SBBQ $0, acc6 |
| SBBQ p256const1<>(SB), acc7 |
| SBBQ $0, hlp |
| // If the result of the subtraction is negative, restore the previous result |
| CMOVQCS acc0, acc4 |
| CMOVQCS acc1, acc5 |
| CMOVQCS acc2, acc6 |
| CMOVQCS acc3, acc7 |
| |
| RET |
| /* ---------------------------------------*/ |
| TEXT p256SqrInternal(SB),NOSPLIT,$0 |
| |
| MOVQ acc4, mul0 |
| MULQ acc5 |
| MOVQ mul0, acc1 |
| MOVQ mul1, acc2 |
| |
| MOVQ acc4, mul0 |
| MULQ acc6 |
| ADDQ mul0, acc2 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc3 |
| |
| MOVQ acc4, mul0 |
| MULQ acc7 |
| ADDQ mul0, acc3 |
| ADCQ $0, mul1 |
| MOVQ mul1, t0 |
| |
| MOVQ acc5, mul0 |
| MULQ acc6 |
| ADDQ mul0, acc3 |
| ADCQ $0, mul1 |
| MOVQ mul1, hlp |
| |
| MOVQ acc5, mul0 |
| MULQ acc7 |
| ADDQ hlp, t0 |
| ADCQ $0, mul1 |
| ADDQ mul0, t0 |
| ADCQ $0, mul1 |
| MOVQ mul1, t1 |
| |
| MOVQ acc6, mul0 |
| MULQ acc7 |
| ADDQ mul0, t1 |
| ADCQ $0, mul1 |
| MOVQ mul1, t2 |
| XORQ t3, t3 |
| // *2 |
| ADDQ acc1, acc1 |
| ADCQ acc2, acc2 |
| ADCQ acc3, acc3 |
| ADCQ t0, t0 |
| ADCQ t1, t1 |
| ADCQ t2, t2 |
| ADCQ $0, t3 |
| // Missing products |
| MOVQ acc4, mul0 |
| MULQ mul0 |
| MOVQ mul0, acc0 |
| MOVQ DX, acc4 |
| |
| MOVQ acc5, mul0 |
| MULQ mul0 |
| ADDQ acc4, acc1 |
| ADCQ mul0, acc2 |
| ADCQ $0, DX |
| MOVQ DX, acc4 |
| |
| MOVQ acc6, mul0 |
| MULQ mul0 |
| ADDQ acc4, acc3 |
| ADCQ mul0, t0 |
| ADCQ $0, DX |
| MOVQ DX, acc4 |
| |
| MOVQ acc7, mul0 |
| MULQ mul0 |
| ADDQ acc4, t1 |
| ADCQ mul0, t2 |
| ADCQ DX, t3 |
| // First reduction step |
| MOVQ acc0, mul0 |
| MOVQ acc0, hlp |
| SHLQ $32, acc0 |
| MULQ p256const1<>(SB) |
| SHRQ $32, hlp |
| ADDQ acc0, acc1 |
| ADCQ hlp, acc2 |
| ADCQ mul0, acc3 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc0 |
| // Second reduction step |
| MOVQ acc1, mul0 |
| MOVQ acc1, hlp |
| SHLQ $32, acc1 |
| MULQ p256const1<>(SB) |
| SHRQ $32, hlp |
| ADDQ acc1, acc2 |
| ADCQ hlp, acc3 |
| ADCQ mul0, acc0 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc1 |
| // Third reduction step |
| MOVQ acc2, mul0 |
| MOVQ acc2, hlp |
| SHLQ $32, acc2 |
| MULQ p256const1<>(SB) |
| SHRQ $32, hlp |
| ADDQ acc2, acc3 |
| ADCQ hlp, acc0 |
| ADCQ mul0, acc1 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc2 |
| // Last reduction step |
| MOVQ acc3, mul0 |
| MOVQ acc3, hlp |
| SHLQ $32, acc3 |
| MULQ p256const1<>(SB) |
| SHRQ $32, hlp |
| ADDQ acc3, acc0 |
| ADCQ hlp, acc1 |
| ADCQ mul0, acc2 |
| ADCQ $0, mul1 |
| MOVQ mul1, acc3 |
| BYTE $0x48; BYTE $0xc7; BYTE $0xc5; BYTE $0x00; BYTE $0x00; BYTE $0x00; BYTE $0x00 // MOVQ $0, BP |
| // Add bits [511:256] of the result |
| ADCQ acc0, t0 |
| ADCQ acc1, t1 |
| ADCQ acc2, t2 |
| ADCQ acc3, t3 |
| ADCQ $0, hlp |
| // Copy result |
| MOVQ t0, acc4 |
| MOVQ t1, acc5 |
| MOVQ t2, acc6 |
| MOVQ t3, acc7 |
| // Subtract p256 |
| SUBQ $-1, acc4 |
| SBBQ p256const0<>(SB) ,acc5 |
| SBBQ $0, acc6 |
| SBBQ p256const1<>(SB), acc7 |
| SBBQ $0, hlp |
| // If the result of the subtraction is negative, restore the previous result |
| CMOVQCS t0, acc4 |
| CMOVQCS t1, acc5 |
| CMOVQCS t2, acc6 |
| CMOVQCS t3, acc7 |
| |
| RET |
| /* ---------------------------------------*/ |
| #define p256MulBy2Inline\ |
| XORQ mul0, mul0;\ |
| ADDQ acc4, acc4;\ |
| ADCQ acc5, acc5;\ |
| ADCQ acc6, acc6;\ |
| ADCQ acc7, acc7;\ |
| ADCQ $0, mul0;\ |
| MOVQ acc4, t0;\ |
| MOVQ acc5, t1;\ |
| MOVQ acc6, t2;\ |
| MOVQ acc7, t3;\ |
| SUBQ $-1, t0;\ |
| SBBQ p256const0<>(SB), t1;\ |
| SBBQ $0, t2;\ |
| SBBQ p256const1<>(SB), t3;\ |
| SBBQ $0, mul0;\ |
| CMOVQCS acc4, t0;\ |
| CMOVQCS acc5, t1;\ |
| CMOVQCS acc6, t2;\ |
| CMOVQCS acc7, t3; |
| /* ---------------------------------------*/ |
| #define p256AddInline \ |
| XORQ mul0, mul0;\ |
| ADDQ t0, acc4;\ |
| ADCQ t1, acc5;\ |
| ADCQ t2, acc6;\ |
| ADCQ t3, acc7;\ |
| ADCQ $0, mul0;\ |
| MOVQ acc4, t0;\ |
| MOVQ acc5, t1;\ |
| MOVQ acc6, t2;\ |
| MOVQ acc7, t3;\ |
| SUBQ $-1, t0;\ |
| SBBQ p256const0<>(SB), t1;\ |
| SBBQ $0, t2;\ |
| SBBQ p256const1<>(SB), t3;\ |
| SBBQ $0, mul0;\ |
| CMOVQCS acc4, t0;\ |
| CMOVQCS acc5, t1;\ |
| CMOVQCS acc6, t2;\ |
| CMOVQCS acc7, t3; |
| /* ---------------------------------------*/ |
| #define LDacc(src) MOVQ src(8*0), acc4; MOVQ src(8*1), acc5; MOVQ src(8*2), acc6; MOVQ src(8*3), acc7 |
| #define LDt(src) MOVQ src(8*0), t0; MOVQ src(8*1), t1; MOVQ src(8*2), t2; MOVQ src(8*3), t3 |
| #define ST(dst) MOVQ acc4, dst(8*0); MOVQ acc5, dst(8*1); MOVQ acc6, dst(8*2); MOVQ acc7, dst(8*3) |
| #define STt(dst) MOVQ t0, dst(8*0); MOVQ t1, dst(8*1); MOVQ t2, dst(8*2); MOVQ t3, dst(8*3) |
| #define acc2t MOVQ acc4, t0; MOVQ acc5, t1; MOVQ acc6, t2; MOVQ acc7, t3 |
| #define t2acc MOVQ t0, acc4; MOVQ t1, acc5; MOVQ t2, acc6; MOVQ t3, acc7 |
| /* ---------------------------------------*/ |
| #define x1in(off) (32*0 + off)(SP) |
| #define y1in(off) (32*1 + off)(SP) |
| #define z1in(off) (32*2 + off)(SP) |
| #define x2in(off) (32*3 + off)(SP) |
| #define y2in(off) (32*4 + off)(SP) |
| #define xout(off) (32*5 + off)(SP) |
| #define yout(off) (32*6 + off)(SP) |
| #define zout(off) (32*7 + off)(SP) |
| #define s2(off) (32*8 + off)(SP) |
| #define z1sqr(off) (32*9 + off)(SP) |
| #define h(off) (32*10 + off)(SP) |
| #define r(off) (32*11 + off)(SP) |
| #define hsqr(off) (32*12 + off)(SP) |
| #define rsqr(off) (32*13 + off)(SP) |
| #define hcub(off) (32*14 + off)(SP) |
| #define rptr (32*15)(SP) |
| #define sel_save (32*15 + 8)(SP) |
| #define zero_save (32*15 + 8 + 4)(SP) |
| |
| // func p256PointAddAffineAsm(res, in1, in2 []uint64, sign, sel, zero int) |
| TEXT ·p256PointAddAffineAsm(SB),0,$512-96 |
| // Move input to stack in order to free registers |
| MOVQ res+0(FP), AX |
| MOVQ in1+24(FP), BX |
| MOVQ in2+48(FP), CX |
| MOVQ sign+72(FP), DX |
| MOVQ sel+80(FP), t1 |
| MOVQ zero+88(FP), t2 |
| |
| MOVOU (16*0)(BX), X0 |
| MOVOU (16*1)(BX), X1 |
| MOVOU (16*2)(BX), X2 |
| MOVOU (16*3)(BX), X3 |
| MOVOU (16*4)(BX), X4 |
| MOVOU (16*5)(BX), X5 |
| |
| MOVOU X0, x1in(16*0) |
| MOVOU X1, x1in(16*1) |
| MOVOU X2, y1in(16*0) |
| MOVOU X3, y1in(16*1) |
| MOVOU X4, z1in(16*0) |
| MOVOU X5, z1in(16*1) |
| |
| MOVOU (16*0)(CX), X0 |
| MOVOU (16*1)(CX), X1 |
| |
| MOVOU X0, x2in(16*0) |
| MOVOU X1, x2in(16*1) |
| // Store pointer to result |
| MOVQ mul0, rptr |
| MOVL t1, sel_save |
| MOVL t2, zero_save |
| // Negate y2in based on sign |
| MOVQ (16*2 + 8*0)(CX), acc4 |
| MOVQ (16*2 + 8*1)(CX), acc5 |
| MOVQ (16*2 + 8*2)(CX), acc6 |
| MOVQ (16*2 + 8*3)(CX), acc7 |
| MOVQ $-1, acc0 |
| MOVQ p256const0<>(SB), acc1 |
| MOVQ $0, acc2 |
| MOVQ p256const1<>(SB), acc3 |
| XORQ mul0, mul0 |
| // Speculatively subtract |
| SUBQ acc4, acc0 |
| SBBQ acc5, acc1 |
| SBBQ acc6, acc2 |
| SBBQ acc7, acc3 |
| SBBQ $0, mul0 |
| MOVQ acc0, t0 |
| MOVQ acc1, t1 |
| MOVQ acc2, t2 |
| MOVQ acc3, t3 |
| // Add in case the operand was > p256 |
| ADDQ $-1, acc0 |
| ADCQ p256const0<>(SB), acc1 |
| ADCQ $0, acc2 |
| ADCQ p256const1<>(SB), acc3 |
| ADCQ $0, mul0 |
| CMOVQNE t0, acc0 |
| CMOVQNE t1, acc1 |
| CMOVQNE t2, acc2 |
| CMOVQNE t3, acc3 |
| // If condition is 0, keep original value |
| TESTQ DX, DX |
| CMOVQEQ acc4, acc0 |
| CMOVQEQ acc5, acc1 |
| CMOVQEQ acc6, acc2 |
| CMOVQEQ acc7, acc3 |
| // Store result |
| MOVQ acc0, y2in(8*0) |
| MOVQ acc1, y2in(8*1) |
| MOVQ acc2, y2in(8*2) |
| MOVQ acc3, y2in(8*3) |
| // Begin point add |
| LDacc (z1in) |
| CALL p256SqrInternal(SB) // z1ˆ2 |
| ST (z1sqr) |
| |
| LDt (x2in) |
| CALL p256MulInternal(SB) // x2 * z1ˆ2 |
| |
| LDt (x1in) |
| CALL p256SubInternal(SB) // h = u2 - u1 |
| ST (h) |
| |
| LDt (z1in) |
| CALL p256MulInternal(SB) // z3 = h * z1 |
| ST (zout) |
| |
| LDacc (z1sqr) |
| CALL p256MulInternal(SB) // z1ˆ3 |
| |
| LDt (y2in) |
| CALL p256MulInternal(SB) // s2 = y2 * z1ˆ3 |
| ST (s2) |
| |
| LDt (y1in) |
| CALL p256SubInternal(SB) // r = s2 - s1 |
| ST (r) |
| |
| CALL p256SqrInternal(SB) // rsqr = rˆ2 |
| ST (rsqr) |
| |
| LDacc (h) |
| CALL p256SqrInternal(SB) // hsqr = hˆ2 |
| ST (hsqr) |
| |
| LDt (h) |
| CALL p256MulInternal(SB) // hcub = hˆ3 |
| ST (hcub) |
| |
| LDt (y1in) |
| CALL p256MulInternal(SB) // y1 * hˆ3 |
| ST (s2) |
| |
| LDacc (x1in) |
| LDt (hsqr) |
| CALL p256MulInternal(SB) // u1 * hˆ2 |
| ST (h) |
| |
| p256MulBy2Inline // u1 * hˆ2 * 2, inline |
| LDacc (rsqr) |
| CALL p256SubInternal(SB) // rˆ2 - u1 * hˆ2 * 2 |
| |
| LDt (hcub) |
| CALL p256SubInternal(SB) |
| ST (xout) |
| |
| MOVQ acc4, t0 |
| MOVQ acc5, t1 |
| MOVQ acc6, t2 |
| MOVQ acc7, t3 |
| LDacc (h) |
| CALL p256SubInternal(SB) |
| |
| LDt (r) |
| CALL p256MulInternal(SB) |
| |
| LDt (s2) |
| CALL p256SubInternal(SB) |
| ST (yout) |
| // Load stored values from stack |
| MOVQ rptr, AX |
| MOVL sel_save, BX |
| MOVL zero_save, CX |
| // The result is not valid if (sel == 0), conditional choose |
| MOVOU xout(16*0), X0 |
| MOVOU xout(16*1), X1 |
| MOVOU yout(16*0), X2 |
| MOVOU yout(16*1), X3 |
| MOVOU zout(16*0), X4 |
| MOVOU zout(16*1), X5 |
| |
| MOVL BX, X6 |
| MOVL CX, X7 |
| |
| PXOR X8, X8 |
| PCMPEQL X9, X9 |
| |
| PSHUFD $0, X6, X6 |
| PSHUFD $0, X7, X7 |
| |
| PCMPEQL X8, X6 |
| PCMPEQL X8, X7 |
| |
| MOVOU X6, X15 |
| PANDN X9, X15 |
| |
| MOVOU x1in(16*0), X9 |
| MOVOU x1in(16*1), X10 |
| MOVOU y1in(16*0), X11 |
| MOVOU y1in(16*1), X12 |
| MOVOU z1in(16*0), X13 |
| MOVOU z1in(16*1), X14 |
| |
| PAND X15, X0 |
| PAND X15, X1 |
| PAND X15, X2 |
| PAND X15, X3 |
| PAND X15, X4 |
| PAND X15, X5 |
| |
| PAND X6, X9 |
| PAND X6, X10 |
| PAND X6, X11 |
| PAND X6, X12 |
| PAND X6, X13 |
| PAND X6, X14 |
| |
| PXOR X9, X0 |
| PXOR X10, X1 |
| PXOR X11, X2 |
| PXOR X12, X3 |
| PXOR X13, X4 |
| PXOR X14, X5 |
| // Similarly if zero == 0 |
| PCMPEQL X9, X9 |
| MOVOU X7, X15 |
| PANDN X9, X15 |
| |
| MOVOU x2in(16*0), X9 |
| MOVOU x2in(16*1), X10 |
| MOVOU y2in(16*0), X11 |
| MOVOU y2in(16*1), X12 |
| MOVOU p256one<>+0x00(SB), X13 |
| MOVOU p256one<>+0x10(SB), X14 |
| |
| PAND X15, X0 |
| PAND X15, X1 |
| PAND X15, X2 |
| PAND X15, X3 |
| PAND X15, X4 |
| PAND X15, X5 |
| |
| PAND X7, X9 |
| PAND X7, X10 |
| PAND X7, X11 |
| PAND X7, X12 |
| PAND X7, X13 |
| PAND X7, X14 |
| |
| PXOR X9, X0 |
| PXOR X10, X1 |
| PXOR X11, X2 |
| PXOR X12, X3 |
| PXOR X13, X4 |
| PXOR X14, X5 |
| // Finally output the result |
| MOVOU X0, (16*0)(AX) |
| MOVOU X1, (16*1)(AX) |
| MOVOU X2, (16*2)(AX) |
| MOVOU X3, (16*3)(AX) |
| MOVOU X4, (16*4)(AX) |
| MOVOU X5, (16*5)(AX) |
| MOVQ $0, rptr |
| |
| RET |
| #undef x1in |
| #undef y1in |
| #undef z1in |
| #undef x2in |
| #undef y2in |
| #undef xout |
| #undef yout |
| #undef zout |
| #undef s2 |
| #undef z1sqr |
| #undef h |
| #undef r |
| #undef hsqr |
| #undef rsqr |
| #undef hcub |
| #undef rptr |
| #undef sel_save |
| #undef zero_save |
| /* ---------------------------------------*/ |
| #define x1in(off) (32*0 + off)(SP) |
| #define y1in(off) (32*1 + off)(SP) |
| #define z1in(off) (32*2 + off)(SP) |
| #define x2in(off) (32*3 + off)(SP) |
| #define y2in(off) (32*4 + off)(SP) |
| #define z2in(off) (32*5 + off)(SP) |
| |
| #define xout(off) (32*6 + off)(SP) |
| #define yout(off) (32*7 + off)(SP) |
| #define zout(off) (32*8 + off)(SP) |
| |
| #define u1(off) (32*9 + off)(SP) |
| #define u2(off) (32*10 + off)(SP) |
| #define s1(off) (32*11 + off)(SP) |
| #define s2(off) (32*12 + off)(SP) |
| #define z1sqr(off) (32*13 + off)(SP) |
| #define z2sqr(off) (32*14 + off)(SP) |
| #define h(off) (32*15 + off)(SP) |
| #define r(off) (32*16 + off)(SP) |
| #define hsqr(off) (32*17 + off)(SP) |
| #define rsqr(off) (32*18 + off)(SP) |
| #define hcub(off) (32*19 + off)(SP) |
| #define rptr (32*20)(SP) |
| |
| //func p256PointAddAsm(res, in1, in2 []uint64) |
| TEXT ·p256PointAddAsm(SB),0,$672-72 |
| // Move input to stack in order to free registers |
| MOVQ res+0(FP), AX |
| MOVQ in1+24(FP), BX |
| MOVQ in2+48(FP), CX |
| |
| MOVOU (16*0)(BX), X0 |
| MOVOU (16*1)(BX), X1 |
| MOVOU (16*2)(BX), X2 |
| MOVOU (16*3)(BX), X3 |
| MOVOU (16*4)(BX), X4 |
| MOVOU (16*5)(BX), X5 |
| |
| MOVOU X0, x1in(16*0) |
| MOVOU X1, x1in(16*1) |
| MOVOU X2, y1in(16*0) |
| MOVOU X3, y1in(16*1) |
| MOVOU X4, z1in(16*0) |
| MOVOU X5, z1in(16*1) |
| |
| MOVOU (16*0)(CX), X0 |
| MOVOU (16*1)(CX), X1 |
| MOVOU (16*2)(CX), X2 |
| MOVOU (16*3)(CX), X3 |
| MOVOU (16*4)(CX), X4 |
| MOVOU (16*5)(CX), X5 |
| |
| MOVOU X0, x2in(16*0) |
| MOVOU X1, x2in(16*1) |
| MOVOU X2, y2in(16*0) |
| MOVOU X3, y2in(16*1) |
| MOVOU X4, z2in(16*0) |
| MOVOU X5, z2in(16*1) |
| // Store pointer to result |
| MOVQ AX, rptr |
| // Begin point add |
| LDacc (z2in) |
| CALL p256SqrInternal(SB) // z2ˆ2 |
| ST (z2sqr) |
| LDt (z2in) |
| CALL p256MulInternal(SB) // z2ˆ3 |
| LDt (y1in) |
| CALL p256MulInternal(SB) // s1 = z2ˆ3*y1 |
| ST (s1) |
| |
| LDacc (z1in) |
| CALL p256SqrInternal(SB) // z1ˆ2 |
| ST (z1sqr) |
| LDt (z1in) |
| CALL p256MulInternal(SB) // z1ˆ3 |
| LDt (y2in) |
| CALL p256MulInternal(SB) // s2 = z1ˆ3*y2 |
| ST (s2) |
| |
| LDt (s1) |
| CALL p256SubInternal(SB) // r = s2 - s1 |
| ST (r) |
| |
| LDacc (z2sqr) |
| LDt (x1in) |
| CALL p256MulInternal(SB) // u1 = x1 * z2ˆ2 |
| ST (u1) |
| LDacc (z1sqr) |
| LDt (x2in) |
| CALL p256MulInternal(SB) // u2 = x2 * z1ˆ2 |
| ST (u2) |
| |
| LDt (u1) |
| CALL p256SubInternal(SB) // h = u2 - u1 |
| ST (h) |
| |
| LDacc (r) |
| CALL p256SqrInternal(SB) // rsqr = rˆ2 |
| ST (rsqr) |
| |
| LDacc (h) |
| CALL p256SqrInternal(SB) // hsqr = hˆ2 |
| ST (hsqr) |
| |
| LDt (h) |
| CALL p256MulInternal(SB) // hcub = hˆ3 |
| ST (hcub) |
| |
| LDt (s1) |
| CALL p256MulInternal(SB) |
| ST (s2) |
| |
| LDacc (z1in) |
| LDt (z2in) |
| CALL p256MulInternal(SB) // z1 * z2 |
| LDt (h) |
| CALL p256MulInternal(SB) // z1 * z2 * h |
| ST (zout) |
| |
| LDacc (hsqr) |
| LDt (u1) |
| CALL p256MulInternal(SB) // hˆ2 * u1 |
| ST (u2) |
| |
| p256MulBy2Inline // u1 * hˆ2 * 2, inline |
| LDacc (rsqr) |
| CALL p256SubInternal(SB) // rˆ2 - u1 * hˆ2 * 2 |
| |
| LDt (hcub) |
| CALL p256SubInternal(SB) |
| ST (xout) |
| |
| MOVQ acc4, t0 |
| MOVQ acc5, t1 |
| MOVQ acc6, t2 |
| MOVQ acc7, t3 |
| LDacc (u2) |
| CALL p256SubInternal(SB) |
| |
| LDt (r) |
| CALL p256MulInternal(SB) |
| |
| LDt (s2) |
| CALL p256SubInternal(SB) |
| ST (yout) |
| |
| MOVOU xout(16*0), X0 |
| MOVOU xout(16*1), X1 |
| MOVOU yout(16*0), X2 |
| MOVOU yout(16*1), X3 |
| MOVOU zout(16*0), X4 |
| MOVOU zout(16*1), X5 |
| // Finally output the result |
| MOVQ rptr, AX |
| MOVQ $0, rptr |
| MOVOU X0, (16*0)(AX) |
| MOVOU X1, (16*1)(AX) |
| MOVOU X2, (16*2)(AX) |
| MOVOU X3, (16*3)(AX) |
| MOVOU X4, (16*4)(AX) |
| MOVOU X5, (16*5)(AX) |
| |
| RET |
| #undef x1in |
| #undef y1in |
| #undef z1in |
| #undef x2in |
| #undef y2in |
| #undef z2in |
| #undef xout |
| #undef yout |
| #undef zout |
| #undef s1 |
| #undef s2 |
| #undef u1 |
| #undef u2 |
| #undef z1sqr |
| #undef z2sqr |
| #undef h |
| #undef r |
| #undef hsqr |
| #undef rsqr |
| #undef hcub |
| #undef rptr |
| /* ---------------------------------------*/ |
| #define x(off) (32*0 + off)(SP) |
| #define y(off) (32*1 + off)(SP) |
| #define z(off) (32*2 + off)(SP) |
| |
| #define s(off) (32*3 + off)(SP) |
| #define m(off) (32*4 + off)(SP) |
| #define zsqr(off) (32*5 + off)(SP) |
| #define tmp(off) (32*6 + off)(SP) |
| #define rptr (32*7)(SP) |
| |
| //func p256PointDoubleAsm(res, in []uint64) |
| TEXT ·p256PointDoubleAsm(SB),NOSPLIT,$256-48 |
| // Move input to stack in order to free registers |
| MOVQ res+0(FP), AX |
| MOVQ in+24(FP), BX |
| |
| MOVOU (16*0)(BX), X0 |
| MOVOU (16*1)(BX), X1 |
| MOVOU (16*2)(BX), X2 |
| MOVOU (16*3)(BX), X3 |
| MOVOU (16*4)(BX), X4 |
| MOVOU (16*5)(BX), X5 |
| |
| MOVOU X0, x(16*0) |
| MOVOU X1, x(16*1) |
| MOVOU X2, y(16*0) |
| MOVOU X3, y(16*1) |
| MOVOU X4, z(16*0) |
| MOVOU X5, z(16*1) |
| // Store pointer to result |
| MOVQ AX, rptr |
| // Begin point double |
| LDacc (z) |
| CALL p256SqrInternal(SB) |
| ST (zsqr) |
| |
| LDt (x) |
| p256AddInline |
| STt (m) |
| |
| LDacc (z) |
| LDt (y) |
| CALL p256MulInternal(SB) |
| p256MulBy2Inline |
| MOVQ rptr, AX |
| // Store z |
| MOVQ t0, (16*4 + 8*0)(AX) |
| MOVQ t1, (16*4 + 8*1)(AX) |
| MOVQ t2, (16*4 + 8*2)(AX) |
| MOVQ t3, (16*4 + 8*3)(AX) |
| |
| LDacc (x) |
| LDt (zsqr) |
| CALL p256SubInternal(SB) |
| LDt (m) |
| CALL p256MulInternal(SB) |
| ST (m) |
| // Multiply by 3 |
| p256MulBy2Inline |
| LDacc (m) |
| p256AddInline |
| STt (m) |
| //////////////////////// |
| LDacc (y) |
| p256MulBy2Inline |
| t2acc |
| CALL p256SqrInternal(SB) |
| ST (s) |
| CALL p256SqrInternal(SB) |
| // Divide by 2 |
| XORQ mul0, mul0 |
| MOVQ acc4, t0 |
| MOVQ acc5, t1 |
| MOVQ acc6, t2 |
| MOVQ acc7, t3 |
| |
| ADDQ $-1, acc4 |
| ADCQ p256const0<>(SB), acc5 |
| ADCQ $0, acc6 |
| ADCQ p256const1<>(SB), acc7 |
| ADCQ $0, mul0 |
| TESTQ $1, t0 |
| |
| CMOVQEQ t0, acc4 |
| CMOVQEQ t1, acc5 |
| CMOVQEQ t2, acc6 |
| CMOVQEQ t3, acc7 |
| ANDQ t0, mul0 |
| |
| SHRQ $1, acc4:acc5 |
| SHRQ $1, acc5:acc6 |
| SHRQ $1, acc6:acc7 |
| SHRQ $1, acc7:mul0 |
| ST (y) |
| ///////////////////////// |
| LDacc (x) |
| LDt (s) |
| CALL p256MulInternal(SB) |
| ST (s) |
| p256MulBy2Inline |
| STt (tmp) |
| |
| LDacc (m) |
| CALL p256SqrInternal(SB) |
| LDt (tmp) |
| CALL p256SubInternal(SB) |
| |
| MOVQ rptr, AX |
| // Store x |
| MOVQ acc4, (16*0 + 8*0)(AX) |
| MOVQ acc5, (16*0 + 8*1)(AX) |
| MOVQ acc6, (16*0 + 8*2)(AX) |
| MOVQ acc7, (16*0 + 8*3)(AX) |
| |
| acc2t |
| LDacc (s) |
| CALL p256SubInternal(SB) |
| |
| LDt (m) |
| CALL p256MulInternal(SB) |
| |
| LDt (y) |
| CALL p256SubInternal(SB) |
| MOVQ rptr, AX |
| // Store y |
| MOVQ acc4, (16*2 + 8*0)(AX) |
| MOVQ acc5, (16*2 + 8*1)(AX) |
| MOVQ acc6, (16*2 + 8*2)(AX) |
| MOVQ acc7, (16*2 + 8*3)(AX) |
| /////////////////////// |
| MOVQ $0, rptr |
| |
| RET |
| /* ---------------------------------------*/ |
| |
