| // Copyright 2024 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package main |
| |
| import ( |
| . "github.com/mmcloughlin/avo/build" |
| . "github.com/mmcloughlin/avo/operand" |
| . "github.com/mmcloughlin/avo/reg" |
| _ "golang.org/x/crypto/argon2" |
| ) |
| |
| //go:generate go run . -out ../blamka_amd64.s -pkg argon2 |
| |
| func main() { |
| Package("golang.org/x/crypto/argon2") |
| ConstraintExpr("amd64,gc,!purego") |
| |
| blamkaSSE4() |
| mixBlocksSSE2() |
| xorBlocksSSE2() |
| Generate() |
| } |
| |
| func blamkaSSE4() { |
| Implement("blamkaSSE4") |
| Attributes(NOSPLIT) |
| AllocLocal(0) |
| |
| Load(Param("b"), RAX) |
| |
| c40 := c40_DATA() |
| c48 := c48_DATA() |
| |
| MOVOU(c40, X10) |
| MOVOU(c48, X11) |
| |
| BLAMKA_ROUND_0(AX, 0, X8, X9, X10, X11) |
| BLAMKA_ROUND_0(AX, 16, X8, X9, X10, X11) |
| BLAMKA_ROUND_0(AX, 32, X8, X9, X10, X11) |
| BLAMKA_ROUND_0(AX, 48, X8, X9, X10, X11) |
| BLAMKA_ROUND_0(AX, 64, X8, X9, X10, X11) |
| BLAMKA_ROUND_0(AX, 80, X8, X9, X10, X11) |
| BLAMKA_ROUND_0(AX, 96, X8, X9, X10, X11) |
| BLAMKA_ROUND_0(AX, 112, X8, X9, X10, X11) |
| |
| BLAMKA_ROUND_1(AX, 0, X8, X9, X10, X11) |
| BLAMKA_ROUND_1(AX, 2, X8, X9, X10, X11) |
| BLAMKA_ROUND_1(AX, 4, X8, X9, X10, X11) |
| BLAMKA_ROUND_1(AX, 6, X8, X9, X10, X11) |
| BLAMKA_ROUND_1(AX, 8, X8, X9, X10, X11) |
| BLAMKA_ROUND_1(AX, 10, X8, X9, X10, X11) |
| BLAMKA_ROUND_1(AX, 12, X8, X9, X10, X11) |
| BLAMKA_ROUND_1(AX, 14, X8, X9, X10, X11) |
| RET() |
| } |
| |
| func mixBlocksSSE2() { |
| Implement("mixBlocksSSE2") |
| Attributes(NOSPLIT) |
| AllocLocal(0) |
| |
| Load(Param("out"), RDX) |
| Load(Param("a"), RAX) |
| Load(Param("b"), RBX) |
| Load(Param("c"), RCX) |
| MOVQ(U32(128), RDI) |
| |
| Label("loop") |
| MOVOU(Mem{Base: AX}.Offset(0), X0) |
| MOVOU(Mem{Base: BX}.Offset(0), X1) |
| MOVOU(Mem{Base: CX}.Offset(0), X2) |
| PXOR(X1, X0) |
| PXOR(X2, X0) |
| MOVOU(X0, Mem{Base: DX}.Offset(0)) |
| ADDQ(Imm(16), RAX) |
| ADDQ(Imm(16), RBX) |
| ADDQ(Imm(16), RCX) |
| ADDQ(Imm(16), RDX) |
| SUBQ(Imm(2), RDI) |
| JA(LabelRef("loop")) |
| RET() |
| } |
| |
| func xorBlocksSSE2() { |
| Implement("xorBlocksSSE2") |
| Attributes(NOSPLIT) |
| AllocLocal(0) |
| |
| Load(Param("out"), RDX) |
| Load(Param("a"), RAX) |
| Load(Param("b"), RBX) |
| Load(Param("c"), RCX) |
| MOVQ(U32(128), RDI) |
| |
| Label("loop") |
| MOVOU(Mem{Base: AX}.Offset(0), X0) |
| MOVOU(Mem{Base: BX}.Offset(0), X1) |
| MOVOU(Mem{Base: CX}.Offset(0), X2) |
| MOVOU(Mem{Base: DX}.Offset(0), X3) |
| PXOR(X1, X0) |
| PXOR(X2, X0) |
| PXOR(X3, X0) |
| MOVOU(X0, Mem{Base: DX}.Offset(0)) |
| ADDQ(Imm(16), RAX) |
| ADDQ(Imm(16), RBX) |
| ADDQ(Imm(16), RCX) |
| ADDQ(Imm(16), RDX) |
| SUBQ(Imm(2), RDI) |
| JA(LabelRef("loop")) |
| RET() |
| } |
| |
| func SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) { |
| MOVO(v4, t1) |
| MOVO(v5, v4) |
| MOVO(t1, v5) |
| MOVO(v6, t1) |
| PUNPCKLQDQ(v6, t2) |
| PUNPCKHQDQ(v7, v6) |
| PUNPCKHQDQ(t2, v6) |
| PUNPCKLQDQ(v7, t2) |
| MOVO(t1, v7) |
| MOVO(v2, t1) |
| PUNPCKHQDQ(t2, v7) |
| PUNPCKLQDQ(v3, t2) |
| PUNPCKHQDQ(t2, v2) |
| PUNPCKLQDQ(t1, t2) |
| PUNPCKHQDQ(t2, v3) |
| } |
| |
| func SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) { |
| MOVO(v4, t1) |
| MOVO(v5, v4) |
| MOVO(t1, v5) |
| MOVO(v2, t1) |
| PUNPCKLQDQ(v2, t2) |
| PUNPCKHQDQ(v3, v2) |
| PUNPCKHQDQ(t2, v2) |
| PUNPCKLQDQ(v3, t2) |
| MOVO(t1, v3) |
| MOVO(v6, t1) |
| PUNPCKHQDQ(t2, v3) |
| PUNPCKLQDQ(v7, t2) |
| PUNPCKHQDQ(t2, v6) |
| PUNPCKLQDQ(t1, t2) |
| PUNPCKHQDQ(t2, v7) |
| } |
| |
| func HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, t0, c40, c48 VecPhysical) { |
| MOVO(v0, t0) |
| PMULULQ(v2, t0) |
| PADDQ(v2, v0) |
| PADDQ(t0, v0) |
| PADDQ(t0, v0) |
| PXOR(v0, v6) |
| PSHUFD(Imm(0xB1), v6, v6) |
| MOVO(v4, t0) |
| PMULULQ(v6, t0) |
| PADDQ(v6, v4) |
| PADDQ(t0, v4) |
| PADDQ(t0, v4) |
| PXOR(v4, v2) |
| PSHUFB(c40, v2) |
| MOVO(v0, t0) |
| PMULULQ(v2, t0) |
| PADDQ(v2, v0) |
| PADDQ(t0, v0) |
| PADDQ(t0, v0) |
| PXOR(v0, v6) |
| PSHUFB(c48, v6) |
| MOVO(v4, t0) |
| PMULULQ(v6, t0) |
| PADDQ(v6, v4) |
| PADDQ(t0, v4) |
| PADDQ(t0, v4) |
| PXOR(v4, v2) |
| MOVO(v2, t0) |
| PADDQ(v2, t0) |
| PSRLQ(Imm(63), v2) |
| PXOR(t0, v2) |
| MOVO(v1, t0) |
| PMULULQ(v3, t0) |
| PADDQ(v3, v1) |
| PADDQ(t0, v1) |
| PADDQ(t0, v1) |
| PXOR(v1, v7) |
| PSHUFD(Imm(0xB1), v7, v7) |
| MOVO(v5, t0) |
| PMULULQ(v7, t0) |
| PADDQ(v7, v5) |
| PADDQ(t0, v5) |
| PADDQ(t0, v5) |
| PXOR(v5, v3) |
| PSHUFB(c40, v3) |
| MOVO(v1, t0) |
| PMULULQ(v3, t0) |
| PADDQ(v3, v1) |
| PADDQ(t0, v1) |
| PADDQ(t0, v1) |
| PXOR(v1, v7) |
| PSHUFB(c48, v7) |
| MOVO(v5, t0) |
| PMULULQ(v7, t0) |
| PADDQ(v7, v5) |
| PADDQ(t0, v5) |
| PADDQ(t0, v5) |
| PXOR(v5, v3) |
| MOVO(v3, t0) |
| PADDQ(v3, t0) |
| PSRLQ(Imm(63), v3) |
| PXOR(t0, v3) |
| } |
| |
| func LOAD_MSG_0(block GPPhysical, off int) { |
| var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7} |
| for i, r := range registers { |
| MOVOU(Mem{Base: block}.Offset(8*(off+(i*2))), r) |
| } |
| } |
| |
| func STORE_MSG_0(block GPPhysical, off int) { |
| var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7} |
| for i, r := range registers { |
| MOVOU(r, Mem{Base: block}.Offset(8*(off+(i*2)))) |
| } |
| } |
| |
| func LOAD_MSG_1(block GPPhysical, off int) { |
| var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7} |
| for i, r := range registers { |
| MOVOU(Mem{Base: block}.Offset(8*off+i*16*8), r) |
| } |
| } |
| |
| func STORE_MSG_1(block GPPhysical, off int) { |
| var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7} |
| for i, r := range registers { |
| MOVOU(r, Mem{Base: block}.Offset(8*off+i*16*8)) |
| } |
| } |
| |
| func BLAMKA_ROUND_0(block GPPhysical, off int, t0, t1, c40, c48 VecPhysical) { |
| LOAD_MSG_0(block, off) |
| HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48) |
| SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1) |
| HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48) |
| SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1) |
| STORE_MSG_0(block, off) |
| } |
| |
| func BLAMKA_ROUND_1(block GPPhysical, off int, t0, t1, c40, c48 VecPhysical) { |
| LOAD_MSG_1(block, off) |
| HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48) |
| SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1) |
| HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48) |
| SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1) |
| STORE_MSG_1(block, off) |
| } |
| |
| // ##------------------DATA SECTION-------------------## |
| |
| var c40_DATA_ptr, c48_DATA_ptr *Mem |
| |
| func c40_DATA() Mem { |
| if c40_DATA_ptr != nil { |
| return *c40_DATA_ptr |
| } |
| |
| c40_DATA := GLOBL("·c40", NOPTR|RODATA) |
| c40_DATA_ptr = &c40_DATA |
| DATA(0x00, U64(0x0201000706050403)) |
| DATA(0x08, U64(0x0a09080f0e0d0c0b)) |
| return c40_DATA |
| } |
| func c48_DATA() Mem { |
| if c48_DATA_ptr != nil { |
| return *c48_DATA_ptr |
| } |
| |
| c48_DATA := GLOBL("·c48", NOPTR|RODATA) |
| c48_DATA_ptr = &c48_DATA |
| DATA(0x00, U64(0x0100070605040302)) |
| DATA(0x08, U64(0x09080f0e0d0c0b0a)) |
| return c48_DATA |
| } |
