Kernel/linux-next
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git synced 2025-01-09 07:23:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Browse Source 
Pull crypto updates from Herbert Xu:
 "Algorithms:

   - Fix rmmod crash with x86/curve25519

   - Add ECDH NIST P384

   - Generate assembly files at build-time with perl scripts on arm

   - Switch to HMAC SHA512 DRBG as default DRBG

  Drivers:

   - Add sl3516 crypto engine

   - Add ECDH NIST P384 support in hisilicon/hpre

   - Add {ofb,cfb,ctr} over {aes,sm4} in hisilicon/sec

   - Add {ccm,gcm} over {aes,sm4} in hisilicon/sec

   - Enable omap hwrng driver for TI K3 family

   - Add support for AEAD algorithms in qce"

* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (142 commits)
  crypto: sl3516 - depends on HAS_IOMEM
  crypto: hisilicon/qm - implement for querying hardware tasks status.
  crypto: sl3516 - Fix build warning without CONFIG_PM
  MAINTAINERS: update caam crypto driver maintainers list
  crypto: nx - Fix numerous sparse byte-order warnings
  crypto: nx - Fix RCU warning in nx842_OF_upd_status
  crypto: api - Move crypto attr definitions out of crypto.h
  crypto: nx - Fix memcpy() over-reading in nonce
  crypto: hisilicon/sec - Fix spelling mistake "fallbcak" -> "fallback"
  crypto: sa2ul - Remove unused auth_len variable
  crypto: sl3516 - fix duplicated inclusion
  crypto: hisilicon/zip - adds the max shaper type rate
  crypto: hisilicon/hpre - adds the max shaper type rate
  crypto: hisilicon/sec - adds the max shaper type rate
  crypto: hisilicon/qm - supports to inquiry each function's QoS
  crypto: hisilicon/qm - add pf ping single vf function
  crypto: hisilicon/qm - merges the work initialization process into a single function
  crypto: hisilicon/qm - add the "alg_qos" file node
  crypto: hisilicon/qm - supports writing QoS int the host
  crypto: api - remove CRYPTOA_U32 and related functions
  ...
This commit is contained in:
Linus Torvalds 2021-06-28 16:04:56 -07:00
commit 6159c49e12
108 changed files with 6920 additions and 11011 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
Documentation/devicetree/bindings/crypto/cortina,sl3516-crypto.yaml Normal file
View File

@ -0,0 +1,50 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/crypto/cortina,sl3516-crypto.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: SL3516 cryptographic offloader driver
maintainers:
- Corentin Labbe <clabbe@baylibre.com>
properties:
compatible:
enum:
- cortina,sl3516-crypto
reg:
maxItems: 1
interrupts:
maxItems: 1
clocks:
maxItems: 1
resets:
maxItems: 1
required:
- compatible
- reg
- interrupts
- clocks
- resets
additionalProperties: false
examples:
- |
#include <dt-bindings/interrupt-controller/irq.h>
#include <dt-bindings/clock/cortina,gemini-clock.h>
#include <dt-bindings/reset/cortina,gemini-reset.h>
crypto@62000000 {
compatible = "cortina,sl3516-crypto";
reg = <0x62000000 0x10000>;
interrupts = <7 IRQ_TYPE_EDGE_RISING>;
resets = <&syscon GEMINI_RESET_SECURITY>;
clocks = <&syscon GEMINI_CLK_GATE_SECURITY>;
};

47
Documentation/devicetree/bindings/crypto/intel,ixp4xx-crypto.yaml Normal file
View File

@ -0,0 +1,47 @@
# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
# Copyright 2018 Linaro Ltd.
%YAML 1.2
---
$id: "http://devicetree.org/schemas/crypto/intel,ixp4xx-crypto.yaml#"
$schema: "http://devicetree.org/meta-schemas/core.yaml#"
title: Intel IXP4xx cryptographic engine
maintainers:
- Linus Walleij <linus.walleij@linaro.org>
description: |
The Intel IXP4xx cryptographic engine makes use of the IXP4xx NPE
(Network Processing Engine). Since it is not a device on its own
it is defined as a subnode of the NPE, if crypto support is
available on the platform.
properties:
compatible:
const: intel,ixp4xx-crypto
intel,npe-handle:
$ref: '/schemas/types.yaml#/definitions/phandle-array'
maxItems: 1
description: phandle to the NPE this crypto engine is using, the cell
describing the NPE instance to be used.
queue-rx:
$ref: /schemas/types.yaml#/definitions/phandle-array
maxItems: 1
description: phandle to the RX queue on the NPE, the cell describing
the queue instance to be used.
queue-txready:
$ref: /schemas/types.yaml#/definitions/phandle-array
maxItems: 1
description: phandle to the TX READY queue on the NPE, the cell describing
the queue instance to be used.
required:
- compatible
- intel,npe-handle
- queue-rx
- queue-txready
additionalProperties: false

22
Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml
View File

@ -26,9 +26,16 @@ properties:
reg:
items:
- description: NPE0 register range
- description: NPE1 register range
- description: NPE2 register range
- description: NPE0 (NPE-A) register range
- description: NPE1 (NPE-B) register range
- description: NPE2 (NPE-C) register range
crypto:
$ref: /schemas/crypto/intel,ixp4xx-crypto.yaml#
type: object
description: Optional node for the embedded crypto engine, the node
should be named with the instance number of the NPE engine used for
the crypto engine.
required:
- compatible
@ -38,8 +45,15 @@ additionalProperties: false
examples:
- |
npe@c8006000 {
npe: npe@c8006000 {
compatible = "intel,ixp4xx-network-processing-engine";
reg = <0xc8006000 0x1000>, <0xc8007000 0x1000>, <0xc8008000 0x1000>;
crypto {
compatible = "intel,ixp4xx-crypto";
intel,npe-handle = <&npe 2>;
queue-rx = <&qmgr 30>;
queue-txready = <&qmgr 29>;
};
};
...

23
MAINTAINERS
View File

@ -1811,6 +1811,7 @@ F: Documentation/devicetree/bindings/net/cortina,gemini-ethernet.txt
F: Documentation/devicetree/bindings/pinctrl/cortina,gemini-pinctrl.txt
F: Documentation/devicetree/bindings/rtc/faraday,ftrtc010.txt
F: arch/arm/mach-gemini/
F: drivers/crypto/gemini/
F: drivers/net/ethernet/cortina/
F: drivers/pinctrl/pinctrl-gemini.c
F: drivers/rtc/rtc-ftrtc010.c
@ -1972,6 +1973,7 @@ F: Documentation/devicetree/bindings/interrupt-controller/intel,ixp4xx-interrupt
F: Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml
F: arch/arm/mach-ixp4xx/
F: drivers/clocksource/timer-ixp4xx.c
F: drivers/crypto/ixp4xx_crypto.c
F: drivers/gpio/gpio-ixp4xx.c
F: drivers/irqchip/irq-ixp4xx.c
F: include/linux/irqchip/irq-ixp4xx.h
@ -7179,7 +7181,7 @@ F: include/video/
FREESCALE CAAM (Cryptographic Acceleration and Assurance Module) DRIVER
M: Horia Geantă <horia.geanta@nxp.com>
M: Aymen Sghaier <aymen.sghaier@nxp.com>
M: Pankaj Gupta <pankaj.gupta@nxp.com>
L: linux-crypto@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/crypto/fsl-sec4.txt
@ -7569,6 +7571,12 @@ M: Kieran Bingham <kbingham@kernel.org>
S: Supported
F: scripts/gdb/
GEMINI CRYPTO DRIVER
M: Corentin Labbe <clabbe@baylibre.com>
L: linux-crypto@vger.kernel.org
S: Maintained
F: drivers/crypto/gemini/
GEMTEK FM RADIO RECEIVER DRIVER
M: Hans Verkuil <hverkuil@xs4all.nl>
L: linux-media@vger.kernel.org
@ -9240,6 +9248,12 @@ F: Documentation/admin-guide/media/ipu3_rcb.svg
F: Documentation/userspace-api/media/v4l/pixfmt-meta-intel-ipu3.rst
F: drivers/staging/media/ipu3/
INTEL IXP4XX CRYPTO SUPPORT
M: Corentin Labbe <clabbe@baylibre.com>
L: linux-crypto@vger.kernel.org
S: Maintained
F: drivers/crypto/ixp4xx_crypto.c
INTEL IXP4XX QMGR, NPE, ETHERNET and HSS SUPPORT
M: Krzysztof Halasa <khalasa@piap.pl>
S: Maintained
@ -15143,6 +15157,13 @@ S: Maintained
F: Documentation/devicetree/bindings/opp/qcom-nvmem-cpufreq.txt
F: drivers/cpufreq/qcom-cpufreq-nvmem.c
QUALCOMM CRYPTO DRIVERS
M: Thara Gopinath <thara.gopinath@linaro.org>
L: linux-crypto@vger.kernel.org
L: linux-arm-msm@vger.kernel.org
S: Maintained
F: drivers/crypto/qce/
QUALCOMM EMAC GIGABIT ETHERNET DRIVER
M: Timur Tabi <timur@kernel.org>
L: netdev@vger.kernel.org

10
arch/arm/crypto/Makefile
View File

@ -45,20 +45,12 @@ poly1305-arm-y := poly1305-core.o poly1305-glue.o
nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
curve25519-neon-y := curve25519-core.o curve25519-glue.o
ifdef REGENERATE_ARM_CRYPTO
quiet_cmd_perl = PERL $@
cmd_perl = $(PERL) $(<) > $(@)
$(src)/poly1305-core.S_shipped: $(src)/poly1305-armv4.pl
$(obj)/%-core.S: $(src)/%-armv4.pl
$(call cmd,perl)
$(src)/sha256-core.S_shipped: $(src)/sha256-armv4.pl
$(call cmd,perl)
$(src)/sha512-core.S_shipped: $(src)/sha512-armv4.pl
$(call cmd,perl)
endif
clean-files += poly1305-core.S sha256-core.S sha512-core.S
# massage the perlasm code a bit so we only get the NEON routine if we need it

1158
arch/arm/crypto/poly1305-core.S_shipped
View File

File diff suppressed because it is too large Load Diff

2816
arch/arm/crypto/sha256-core.S_shipped
View File

File diff suppressed because it is too large Load Diff

1869
arch/arm/crypto/sha512-core.S_shipped
View File

File diff suppressed because it is too large Load Diff

26
arch/arm/mach-ixp4xx/common.c
View File

@ -233,12 +233,38 @@ static struct platform_device *ixp46x_devices[] __initdata = {
unsigned long ixp4xx_exp_bus_size;
EXPORT_SYMBOL(ixp4xx_exp_bus_size);
static struct platform_device_info ixp_dev_info __initdata = {
.name = "ixp4xx_crypto",
.id = 0,
.dma_mask = DMA_BIT_MASK(32),
};
static int __init ixp_crypto_register(void)
{
struct platform_device *pdev;
if (!(~(*IXP4XX_EXP_CFG2) & (IXP4XX_FEATURE_HASH |
IXP4XX_FEATURE_AES | IXP4XX_FEATURE_DES))) {
printk(KERN_ERR "ixp_crypto: No HW crypto available\n");
return -ENODEV;
}
pdev = platform_device_register_full(&ixp_dev_info);
if (IS_ERR(pdev))
return PTR_ERR(pdev);
return 0;
}
void __init ixp4xx_sys_init(void)
{
ixp4xx_exp_bus_size = SZ_16M;
platform_add_devices(ixp4xx_devices, ARRAY_SIZE(ixp4xx_devices));
if (IS_ENABLED(CONFIG_CRYPTO_DEV_IXP4XX))
ixp_crypto_register();
if (cpu_is_ixp46x()) {
int region;

10
arch/arm64/crypto/Makefile
View File

@ -68,19 +68,13 @@ CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS
$(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE
$(call if_changed_rule,cc_o_c)
ifdef REGENERATE_ARM64_CRYPTO
quiet_cmd_perlasm = PERLASM $@
cmd_perlasm = $(PERL) $(<) void $(@)
$(src)/poly1305-core.S_shipped: $(src)/poly1305-armv8.pl
$(obj)/%-core.S: $(src)/%-armv8.pl
$(call cmd,perlasm)
$(src)/sha256-core.S_shipped: $(src)/sha512-armv8.pl
$(obj)/sha256-core.S: $(src)/sha512-armv8.pl
$(call cmd,perlasm)
$(src)/sha512-core.S_shipped: $(src)/sha512-armv8.pl
$(call cmd,perlasm)
endif
clean-files += poly1305-core.S sha256-core.S sha512-core.S

835
arch/arm64/crypto/poly1305-core.S_shipped
View File

@ -1,835 +0,0 @@
#ifndef __KERNEL__
# include "arm_arch.h"
.extern OPENSSL_armcap_P
#endif
.text
// forward "declarations" are required for Apple
.globl poly1305_blocks
.globl poly1305_emit
.globl poly1305_init
.type poly1305_init,%function
.align 5
poly1305_init:
cmp x1,xzr
stp xzr,xzr,[x0] // zero hash value
stp xzr,xzr,[x0,#16] // [along with is_base2_26]
csel x0,xzr,x0,eq
b.eq .Lno_key
#ifndef __KERNEL__
adrp x17,OPENSSL_armcap_P
ldr w17,[x17,#:lo12:OPENSSL_armcap_P]
#endif
ldp x7,x8,[x1] // load key
mov x9,#0xfffffffc0fffffff
movk x9,#0x0fff,lsl#48
#ifdef __AARCH64EB__
rev x7,x7 // flip bytes
rev x8,x8
#endif
and x7,x7,x9 // &=0ffffffc0fffffff
and x9,x9,#-4
and x8,x8,x9 // &=0ffffffc0ffffffc
mov w9,#-1
stp x7,x8,[x0,#32] // save key value
str w9,[x0,#48] // impossible key power value
#ifndef __KERNEL__
tst w17,#ARMV7_NEON
adr x12,.Lpoly1305_blocks
adr x7,.Lpoly1305_blocks_neon
adr x13,.Lpoly1305_emit
csel x12,x12,x7,eq
# ifdef __ILP32__
stp w12,w13,[x2]
# else
stp x12,x13,[x2]
# endif
#endif
mov x0,#1
.Lno_key:
ret
.size poly1305_init,.-poly1305_init
.type poly1305_blocks,%function
.align 5
poly1305_blocks:
.Lpoly1305_blocks:
ands x2,x2,#-16
b.eq .Lno_data
ldp x4,x5,[x0] // load hash value
ldp x6,x17,[x0,#16] // [along with is_base2_26]
ldp x7,x8,[x0,#32] // load key value
#ifdef __AARCH64EB__
lsr x12,x4,#32
mov w13,w4
lsr x14,x5,#32
mov w15,w5
lsr x16,x6,#32
#else
mov w12,w4
lsr x13,x4,#32
mov w14,w5
lsr x15,x5,#32
mov w16,w6
#endif
add x12,x12,x13,lsl#26 // base 2^26 -> base 2^64
lsr x13,x14,#12
adds x12,x12,x14,lsl#52
add x13,x13,x15,lsl#14
adc x13,x13,xzr
lsr x14,x16,#24
adds x13,x13,x16,lsl#40
adc x14,x14,xzr
cmp x17,#0 // is_base2_26?
add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2)
csel x4,x4,x12,eq // choose between radixes
csel x5,x5,x13,eq
csel x6,x6,x14,eq
.Loop:
ldp x10,x11,[x1],#16 // load input
sub x2,x2,#16
#ifdef __AARCH64EB__
rev x10,x10
rev x11,x11
#endif
adds x4,x4,x10 // accumulate input
adcs x5,x5,x11
mul x12,x4,x7 // h0*r0
adc x6,x6,x3
umulh x13,x4,x7
mul x10,x5,x9 // h1*5*r1
umulh x11,x5,x9
adds x12,x12,x10
mul x10,x4,x8 // h0*r1
adc x13,x13,x11
umulh x14,x4,x8
adds x13,x13,x10
mul x10,x5,x7 // h1*r0
adc x14,x14,xzr
umulh x11,x5,x7
adds x13,x13,x10
mul x10,x6,x9 // h2*5*r1
adc x14,x14,x11
mul x11,x6,x7 // h2*r0
adds x13,x13,x10
adc x14,x14,x11
and x10,x14,#-4 // final reduction
and x6,x14,#3
add x10,x10,x14,lsr#2
adds x4,x12,x10
adcs x5,x13,xzr
adc x6,x6,xzr
cbnz x2,.Loop
stp x4,x5,[x0] // store hash value
stp x6,xzr,[x0,#16] // [and clear is_base2_26]
.Lno_data:
ret
.size poly1305_blocks,.-poly1305_blocks
.type poly1305_emit,%function
.align 5
poly1305_emit:
.Lpoly1305_emit:
ldp x4,x5,[x0] // load hash base 2^64
ldp x6,x7,[x0,#16] // [along with is_base2_26]
ldp x10,x11,[x2] // load nonce
#ifdef __AARCH64EB__
lsr x12,x4,#32
mov w13,w4
lsr x14,x5,#32
mov w15,w5
lsr x16,x6,#32
#else
mov w12,w4
lsr x13,x4,#32
mov w14,w5
lsr x15,x5,#32
mov w16,w6
#endif
add x12,x12,x13,lsl#26 // base 2^26 -> base 2^64
lsr x13,x14,#12
adds x12,x12,x14,lsl#52
add x13,x13,x15,lsl#14
adc x13,x13,xzr
lsr x14,x16,#24
adds x13,x13,x16,lsl#40
adc x14,x14,xzr
cmp x7,#0 // is_base2_26?
csel x4,x4,x12,eq // choose between radixes
csel x5,x5,x13,eq
csel x6,x6,x14,eq
adds x12,x4,#5 // compare to modulus
adcs x13,x5,xzr
adc x14,x6,xzr
tst x14,#-4 // see if it's carried/borrowed
csel x4,x4,x12,eq
csel x5,x5,x13,eq
#ifdef __AARCH64EB__
ror x10,x10,#32 // flip nonce words
ror x11,x11,#32
#endif
adds x4,x4,x10 // accumulate nonce
adc x5,x5,x11
#ifdef __AARCH64EB__
rev x4,x4 // flip output bytes
rev x5,x5
#endif
stp x4,x5,[x1] // write result
ret
.size poly1305_emit,.-poly1305_emit
.type poly1305_mult,%function
.align 5
poly1305_mult:
mul x12,x4,x7 // h0*r0
umulh x13,x4,x7
mul x10,x5,x9 // h1*5*r1
umulh x11,x5,x9
adds x12,x12,x10
mul x10,x4,x8 // h0*r1
adc x13,x13,x11
umulh x14,x4,x8
adds x13,x13,x10
mul x10,x5,x7 // h1*r0
adc x14,x14,xzr
umulh x11,x5,x7
adds x13,x13,x10
mul x10,x6,x9 // h2*5*r1
adc x14,x14,x11
mul x11,x6,x7 // h2*r0
adds x13,x13,x10
adc x14,x14,x11
and x10,x14,#-4 // final reduction
and x6,x14,#3
add x10,x10,x14,lsr#2
adds x4,x12,x10
adcs x5,x13,xzr
adc x6,x6,xzr
ret
.size poly1305_mult,.-poly1305_mult
.type poly1305_splat,%function
.align 4
poly1305_splat:
and x12,x4,#0x03ffffff // base 2^64 -> base 2^26
ubfx x13,x4,#26,#26
extr x14,x5,x4,#52
and x14,x14,#0x03ffffff
ubfx x15,x5,#14,#26
extr x16,x6,x5,#40
str w12,[x0,#16*0] // r0
add w12,w13,w13,lsl#2 // r1*5
str w13,[x0,#16*1] // r1
add w13,w14,w14,lsl#2 // r2*5
str w12,[x0,#16*2] // s1
str w14,[x0,#16*3] // r2
add w14,w15,w15,lsl#2 // r3*5
str w13,[x0,#16*4] // s2
str w15,[x0,#16*5] // r3
add w15,w16,w16,lsl#2 // r4*5
str w14,[x0,#16*6] // s3
str w16,[x0,#16*7] // r4
str w15,[x0,#16*8] // s4
ret
.size poly1305_splat,.-poly1305_splat
#ifdef __KERNEL__
.globl poly1305_blocks_neon
#endif
.type poly1305_blocks_neon,%function
.align 5
poly1305_blocks_neon:
.Lpoly1305_blocks_neon:
ldr x17,[x0,#24]
cmp x2,#128
b.lo .Lpoly1305_blocks
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-80]!
add x29,sp,#0
stp d8,d9,[sp,#16] // meet ABI requirements
stp d10,d11,[sp,#32]
stp d12,d13,[sp,#48]
stp d14,d15,[sp,#64]
cbz x17,.Lbase2_64_neon
ldp w10,w11,[x0] // load hash value base 2^26
ldp w12,w13,[x0,#8]
ldr w14,[x0,#16]
tst x2,#31
b.eq .Leven_neon
ldp x7,x8,[x0,#32] // load key value
add x4,x10,x11,lsl#26 // base 2^26 -> base 2^64
lsr x5,x12,#12
adds x4,x4,x12,lsl#52
add x5,x5,x13,lsl#14
adc x5,x5,xzr
lsr x6,x14,#24
adds x5,x5,x14,lsl#40
adc x14,x6,xzr // can be partially reduced...
ldp x12,x13,[x1],#16 // load input
sub x2,x2,#16
add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2)
#ifdef __AARCH64EB__
rev x12,x12
rev x13,x13
#endif
adds x4,x4,x12 // accumulate input
adcs x5,x5,x13
adc x6,x6,x3
bl poly1305_mult
and x10,x4,#0x03ffffff // base 2^64 -> base 2^26
ubfx x11,x4,#26,#26
extr x12,x5,x4,#52
and x12,x12,#0x03ffffff
ubfx x13,x5,#14,#26
extr x14,x6,x5,#40
b .Leven_neon
.align 4
.Lbase2_64_neon:
ldp x7,x8,[x0,#32] // load key value
ldp x4,x5,[x0] // load hash value base 2^64
ldr x6,[x0,#16]
tst x2,#31
b.eq .Linit_neon
ldp x12,x13,[x1],#16 // load input
sub x2,x2,#16
add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2)
#ifdef __AARCH64EB__
rev x12,x12
rev x13,x13
#endif
adds x4,x4,x12 // accumulate input
adcs x5,x5,x13
adc x6,x6,x3
bl poly1305_mult
.Linit_neon:
ldr w17,[x0,#48] // first table element
and x10,x4,#0x03ffffff // base 2^64 -> base 2^26
ubfx x11,x4,#26,#26
extr x12,x5,x4,#52
and x12,x12,#0x03ffffff
ubfx x13,x5,#14,#26
extr x14,x6,x5,#40
cmp w17,#-1 // is value impossible?
b.ne .Leven_neon
fmov d24,x10
fmov d25,x11
fmov d26,x12
fmov d27,x13
fmov d28,x14
////////////////////////////////// initialize r^n table
mov x4,x7 // r^1
add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2)
mov x5,x8
mov x6,xzr
add x0,x0,#48+12
bl poly1305_splat
bl poly1305_mult // r^2
sub x0,x0,#4
bl poly1305_splat
bl poly1305_mult // r^3
sub x0,x0,#4
bl poly1305_splat
bl poly1305_mult // r^4
sub x0,x0,#4
bl poly1305_splat
sub x0,x0,#48 // restore original x0
b .Ldo_neon
.align 4
.Leven_neon:
fmov d24,x10
fmov d25,x11
fmov d26,x12
fmov d27,x13
fmov d28,x14
.Ldo_neon:
ldp x8,x12,[x1,#32] // inp[2:3]
subs x2,x2,#64
ldp x9,x13,[x1,#48]
add x16,x1,#96
adr x17,.Lzeros
lsl x3,x3,#24
add x15,x0,#48
#ifdef __AARCH64EB__
rev x8,x8
rev x12,x12
rev x9,x9
rev x13,x13
#endif
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
and x5,x9,#0x03ffffff
ubfx x6,x8,#26,#26
ubfx x7,x9,#26,#26
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
extr x8,x12,x8,#52
extr x9,x13,x9,#52
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
fmov d14,x4
and x8,x8,#0x03ffffff
and x9,x9,#0x03ffffff
ubfx x10,x12,#14,#26
ubfx x11,x13,#14,#26
add x12,x3,x12,lsr#40
add x13,x3,x13,lsr#40
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
fmov d15,x6
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
fmov d16,x8
fmov d17,x10
fmov d18,x12
ldp x8,x12,[x1],#16 // inp[0:1]
ldp x9,x13,[x1],#48
ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x15],#64
ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x15],#64
ld1 {v8.4s},[x15]
#ifdef __AARCH64EB__
rev x8,x8
rev x12,x12
rev x9,x9
rev x13,x13
#endif
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
and x5,x9,#0x03ffffff
ubfx x6,x8,#26,#26
ubfx x7,x9,#26,#26
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
extr x8,x12,x8,#52
extr x9,x13,x9,#52
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
fmov d9,x4
and x8,x8,#0x03ffffff
and x9,x9,#0x03ffffff
ubfx x10,x12,#14,#26
ubfx x11,x13,#14,#26
add x12,x3,x12,lsr#40
add x13,x3,x13,lsr#40
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
fmov d10,x6
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
movi v31.2d,#-1
fmov d11,x8
fmov d12,x10
fmov d13,x12
ushr v31.2d,v31.2d,#38
b.ls .Lskip_loop
.align 4
.Loop_neon:
////////////////////////////////////////////////////////////////
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
// ___________________/
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
// ___________________/ ____________________/
//
// Note that we start with inp[2:3]*r^2. This is because it
// doesn't depend on reduction in previous iteration.
////////////////////////////////////////////////////////////////
// d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0
// d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4
// d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3
// d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2
// d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
subs x2,x2,#64
umull v23.2d,v14.2s,v7.s[2]
csel x16,x17,x16,lo
umull v22.2d,v14.2s,v5.s[2]
umull v21.2d,v14.2s,v3.s[2]
ldp x8,x12,[x16],#16 // inp[2:3] (or zero)
umull v20.2d,v14.2s,v1.s[2]
ldp x9,x13,[x16],#48
umull v19.2d,v14.2s,v0.s[2]
#ifdef __AARCH64EB__
rev x8,x8
rev x12,x12
rev x9,x9
rev x13,x13
#endif
umlal v23.2d,v15.2s,v5.s[2]
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
umlal v22.2d,v15.2s,v3.s[2]
and x5,x9,#0x03ffffff
umlal v21.2d,v15.2s,v1.s[2]
ubfx x6,x8,#26,#26
umlal v20.2d,v15.2s,v0.s[2]
ubfx x7,x9,#26,#26
umlal v19.2d,v15.2s,v8.s[2]
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
umlal v23.2d,v16.2s,v3.s[2]
extr x8,x12,x8,#52
umlal v22.2d,v16.2s,v1.s[2]
extr x9,x13,x9,#52
umlal v21.2d,v16.2s,v0.s[2]
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
umlal v20.2d,v16.2s,v8.s[2]
fmov d14,x4
umlal v19.2d,v16.2s,v6.s[2]
and x8,x8,#0x03ffffff
umlal v23.2d,v17.2s,v1.s[2]
and x9,x9,#0x03ffffff
umlal v22.2d,v17.2s,v0.s[2]
ubfx x10,x12,#14,#26
umlal v21.2d,v17.2s,v8.s[2]
ubfx x11,x13,#14,#26
umlal v20.2d,v17.2s,v6.s[2]
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
umlal v19.2d,v17.2s,v4.s[2]
fmov d15,x6
add v11.2s,v11.2s,v26.2s
add x12,x3,x12,lsr#40
umlal v23.2d,v18.2s,v0.s[2]
add x13,x3,x13,lsr#40
umlal v22.2d,v18.2s,v8.s[2]
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
umlal v21.2d,v18.2s,v6.s[2]
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
umlal v20.2d,v18.2s,v4.s[2]
fmov d16,x8
umlal v19.2d,v18.2s,v2.s[2]
fmov d17,x10
////////////////////////////////////////////////////////////////
// (hash+inp[0:1])*r^4 and accumulate
add v9.2s,v9.2s,v24.2s
fmov d18,x12
umlal v22.2d,v11.2s,v1.s[0]
ldp x8,x12,[x1],#16 // inp[0:1]
umlal v19.2d,v11.2s,v6.s[0]
ldp x9,x13,[x1],#48
umlal v23.2d,v11.2s,v3.s[0]
umlal v20.2d,v11.2s,v8.s[0]
umlal v21.2d,v11.2s,v0.s[0]
#ifdef __AARCH64EB__
rev x8,x8
rev x12,x12
rev x9,x9
rev x13,x13
#endif
add v10.2s,v10.2s,v25.2s
umlal v22.2d,v9.2s,v5.s[0]
umlal v23.2d,v9.2s,v7.s[0]
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
umlal v21.2d,v9.2s,v3.s[0]
and x5,x9,#0x03ffffff
umlal v19.2d,v9.2s,v0.s[0]
ubfx x6,x8,#26,#26
umlal v20.2d,v9.2s,v1.s[0]
ubfx x7,x9,#26,#26
add v12.2s,v12.2s,v27.2s
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
umlal v22.2d,v10.2s,v3.s[0]
extr x8,x12,x8,#52
umlal v23.2d,v10.2s,v5.s[0]
extr x9,x13,x9,#52
umlal v19.2d,v10.2s,v8.s[0]
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
umlal v21.2d,v10.2s,v1.s[0]
fmov d9,x4
umlal v20.2d,v10.2s,v0.s[0]
and x8,x8,#0x03ffffff
add v13.2s,v13.2s,v28.2s
and x9,x9,#0x03ffffff
umlal v22.2d,v12.2s,v0.s[0]
ubfx x10,x12,#14,#26
umlal v19.2d,v12.2s,v4.s[0]
ubfx x11,x13,#14,#26
umlal v23.2d,v12.2s,v1.s[0]
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
umlal v20.2d,v12.2s,v6.s[0]
fmov d10,x6
umlal v21.2d,v12.2s,v8.s[0]
add x12,x3,x12,lsr#40
umlal v22.2d,v13.2s,v8.s[0]
add x13,x3,x13,lsr#40
umlal v19.2d,v13.2s,v2.s[0]
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
umlal v23.2d,v13.2s,v0.s[0]
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
umlal v20.2d,v13.2s,v4.s[0]
fmov d11,x8
umlal v21.2d,v13.2s,v6.s[0]
fmov d12,x10
fmov d13,x12
/////////////////////////////////////////////////////////////////
// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
// and P. Schwabe
//
// [see discussion in poly1305-armv4 module]
ushr v29.2d,v22.2d,#26
xtn v27.2s,v22.2d
ushr v30.2d,v19.2d,#26
and v19.16b,v19.16b,v31.16b
add v23.2d,v23.2d,v29.2d // h3 -> h4
bic v27.2s,#0xfc,lsl#24 // &=0x03ffffff
add v20.2d,v20.2d,v30.2d // h0 -> h1
ushr v29.2d,v23.2d,#26
xtn v28.2s,v23.2d
ushr v30.2d,v20.2d,#26
xtn v25.2s,v20.2d
bic v28.2s,#0xfc,lsl#24
add v21.2d,v21.2d,v30.2d // h1 -> h2
add v19.2d,v19.2d,v29.2d
shl v29.2d,v29.2d,#2
shrn v30.2s,v21.2d,#26
xtn v26.2s,v21.2d
add v19.2d,v19.2d,v29.2d // h4 -> h0
bic v25.2s,#0xfc,lsl#24
add v27.2s,v27.2s,v30.2s // h2 -> h3
bic v26.2s,#0xfc,lsl#24
shrn v29.2s,v19.2d,#26
xtn v24.2s,v19.2d
ushr v30.2s,v27.2s,#26
bic v27.2s,#0xfc,lsl#24
bic v24.2s,#0xfc,lsl#24
add v25.2s,v25.2s,v29.2s // h0 -> h1
add v28.2s,v28.2s,v30.2s // h3 -> h4
b.hi .Loop_neon
.Lskip_loop:
dup v16.2d,v16.d[0]
add v11.2s,v11.2s,v26.2s
////////////////////////////////////////////////////////////////
// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
adds x2,x2,#32
b.ne .Long_tail
dup v16.2d,v11.d[0]
add v14.2s,v9.2s,v24.2s
add v17.2s,v12.2s,v27.2s
add v15.2s,v10.2s,v25.2s
add v18.2s,v13.2s,v28.2s
.Long_tail:
dup v14.2d,v14.d[0]
umull2 v19.2d,v16.4s,v6.4s
umull2 v22.2d,v16.4s,v1.4s
umull2 v23.2d,v16.4s,v3.4s
umull2 v21.2d,v16.4s,v0.4s
umull2 v20.2d,v16.4s,v8.4s
dup v15.2d,v15.d[0]
umlal2 v19.2d,v14.4s,v0.4s
umlal2 v21.2d,v14.4s,v3.4s
umlal2 v22.2d,v14.4s,v5.4s
umlal2 v23.2d,v14.4s,v7.4s
umlal2 v20.2d,v14.4s,v1.4s
dup v17.2d,v17.d[0]
umlal2 v19.2d,v15.4s,v8.4s
umlal2 v22.2d,v15.4s,v3.4s
umlal2 v21.2d,v15.4s,v1.4s
umlal2 v23.2d,v15.4s,v5.4s
umlal2 v20.2d,v15.4s,v0.4s
dup v18.2d,v18.d[0]
umlal2 v22.2d,v17.4s,v0.4s
umlal2 v23.2d,v17.4s,v1.4s
umlal2 v19.2d,v17.4s,v4.4s
umlal2 v20.2d,v17.4s,v6.4s
umlal2 v21.2d,v17.4s,v8.4s
umlal2 v22.2d,v18.4s,v8.4s
umlal2 v19.2d,v18.4s,v2.4s
umlal2 v23.2d,v18.4s,v0.4s
umlal2 v20.2d,v18.4s,v4.4s
umlal2 v21.2d,v18.4s,v6.4s
b.eq .Lshort_tail
////////////////////////////////////////////////////////////////
// (hash+inp[0:1])*r^4:r^3 and accumulate
add v9.2s,v9.2s,v24.2s
umlal v22.2d,v11.2s,v1.2s
umlal v19.2d,v11.2s,v6.2s
umlal v23.2d,v11.2s,v3.2s
umlal v20.2d,v11.2s,v8.2s
umlal v21.2d,v11.2s,v0.2s
add v10.2s,v10.2s,v25.2s
umlal v22.2d,v9.2s,v5.2s
umlal v19.2d,v9.2s,v0.2s
umlal v23.2d,v9.2s,v7.2s
umlal v20.2d,v9.2s,v1.2s
umlal v21.2d,v9.2s,v3.2s
add v12.2s,v12.2s,v27.2s
umlal v22.2d,v10.2s,v3.2s
umlal v19.2d,v10.2s,v8.2s
umlal v23.2d,v10.2s,v5.2s
umlal v20.2d,v10.2s,v0.2s
umlal v21.2d,v10.2s,v1.2s
add v13.2s,v13.2s,v28.2s
umlal v22.2d,v12.2s,v0.2s
umlal v19.2d,v12.2s,v4.2s
umlal v23.2d,v12.2s,v1.2s
umlal v20.2d,v12.2s,v6.2s
umlal v21.2d,v12.2s,v8.2s
umlal v22.2d,v13.2s,v8.2s
umlal v19.2d,v13.2s,v2.2s
umlal v23.2d,v13.2s,v0.2s
umlal v20.2d,v13.2s,v4.2s
umlal v21.2d,v13.2s,v6.2s
.Lshort_tail:
////////////////////////////////////////////////////////////////
// horizontal add
addp v22.2d,v22.2d,v22.2d
ldp d8,d9,[sp,#16] // meet ABI requirements
addp v19.2d,v19.2d,v19.2d
ldp d10,d11,[sp,#32]
addp v23.2d,v23.2d,v23.2d
ldp d12,d13,[sp,#48]
addp v20.2d,v20.2d,v20.2d
ldp d14,d15,[sp,#64]
addp v21.2d,v21.2d,v21.2d
ldr x30,[sp,#8]
////////////////////////////////////////////////////////////////
// lazy reduction, but without narrowing
ushr v29.2d,v22.2d,#26
and v22.16b,v22.16b,v31.16b
ushr v30.2d,v19.2d,#26
and v19.16b,v19.16b,v31.16b
add v23.2d,v23.2d,v29.2d // h3 -> h4
add v20.2d,v20.2d,v30.2d // h0 -> h1
ushr v29.2d,v23.2d,#26
and v23.16b,v23.16b,v31.16b
ushr v30.2d,v20.2d,#26
and v20.16b,v20.16b,v31.16b
add v21.2d,v21.2d,v30.2d // h1 -> h2
add v19.2d,v19.2d,v29.2d
shl v29.2d,v29.2d,#2
ushr v30.2d,v21.2d,#26
and v21.16b,v21.16b,v31.16b
add v19.2d,v19.2d,v29.2d // h4 -> h0
add v22.2d,v22.2d,v30.2d // h2 -> h3
ushr v29.2d,v19.2d,#26
and v19.16b,v19.16b,v31.16b
ushr v30.2d,v22.2d,#26
and v22.16b,v22.16b,v31.16b
add v20.2d,v20.2d,v29.2d // h0 -> h1
add v23.2d,v23.2d,v30.2d // h3 -> h4
////////////////////////////////////////////////////////////////
// write the result, can be partially reduced
st4 {v19.s,v20.s,v21.s,v22.s}[0],[x0],#16
mov x4,#1
st1 {v23.s}[0],[x0]
str x4,[x0,#8] // set is_base2_26
ldr x29,[sp],#80
.inst 0xd50323bf // autiasp
ret
.size poly1305_blocks_neon,.-poly1305_blocks_neon
.align 5
.Lzeros:
.long 0,0,0,0,0,0,0,0
.asciz "Poly1305 for ARMv8, CRYPTOGAMS by @dot-asm"
.align 2
#if !defined(__KERNEL__) && !defined(_WIN64)
.comm OPENSSL_armcap_P,4,4
.hidden OPENSSL_armcap_P
#endif

2069
arch/arm64/crypto/sha256-core.S_shipped
View File

File diff suppressed because it is too large Load Diff

1093
arch/arm64/crypto/sha512-core.S_shipped
View File

File diff suppressed because it is too large Load Diff

2
arch/x86/crypto/curve25519-x86_64.c
View File

@ -1500,7 +1500,7 @@ static int __init curve25519_mod_init(void)
static void __exit curve25519_mod_exit(void)
{
if (IS_REACHABLE(CONFIG_CRYPTO_KPP) &&
(boot_cpu_has(X86_FEATURE_BMI2) || boot_cpu_has(X86_FEATURE_ADX)))
static_branch_likely(&curve25519_use_bmi2_adx))
crypto_unregister_kpp(&curve25519_alg);
}

2
crypto/af_alg.c
View File

@ -411,7 +411,7 @@ int af_alg_make_sg(struct af_alg_sgl *sgl, struct iov_iter *iter, int len)
if (n < 0)
return n;
npages = (off + n + PAGE_SIZE - 1) >> PAGE_SHIFT;
npages = DIV_ROUND_UP(off + n, PAGE_SIZE);
if (WARN_ON(npages == 0))
return -EINVAL;
/* Add one extra for linking */

18
crypto/algapi.c
View File

@ -868,24 +868,6 @@ const char *crypto_attr_alg_name(struct rtattr *rta)
}
EXPORT_SYMBOL_GPL(crypto_attr_alg_name);
int crypto_attr_u32(struct rtattr *rta, u32 *num)
{
struct crypto_attr_u32 *nu32;
if (!rta)
return -ENOENT;
if (RTA_PAYLOAD(rta) < sizeof(*nu32))
return -EINVAL;
if (rta->rta_type != CRYPTOA_U32)
return -EINVAL;
nu32 = RTA_DATA(rta);
*num = nu32->num;
return 0;
}
EXPORT_SYMBOL_GPL(crypto_attr_u32);
int crypto_inst_setname(struct crypto_instance *inst, const char *name,
struct crypto_alg *alg)
{

31
crypto/algboss.c
View File

@ -28,16 +28,9 @@ struct cryptomgr_param {
struct crypto_attr_type data;
} type;
union {
struct {
struct rtattr attr;
struct {
struct rtattr attr;
struct crypto_attr_alg data;
} alg;
struct {
struct rtattr attr;
struct crypto_attr_u32 data;
} nu32;
struct crypto_attr_alg data;
} attrs[CRYPTO_MAX_ATTRS];
char template[CRYPTO_MAX_ALG_NAME];
@ -104,12 +97,10 @@ static int cryptomgr_schedule_probe(struct crypto_larval *larval)
i = 0;
for (;;) {
int notnum = 0;
name = ++p;
for (; isalnum(*p) || *p == '-' || *p == '_'; p++)
notnum |= !isdigit(*p);
;
if (*p == '(') {
int recursion = 0;
@ -123,7 +114,6 @@ static int cryptomgr_schedule_probe(struct crypto_larval *larval)
break;
}
notnum = 1;
p++;
}
@ -131,18 +121,9 @@ static int cryptomgr_schedule_probe(struct crypto_larval *larval)
if (!len)
goto err_free_param;
if (notnum) {
param->attrs[i].alg.attr.rta_len =
sizeof(param->attrs[i].alg);
param->attrs[i].alg.attr.rta_type = CRYPTOA_ALG;
memcpy(param->attrs[i].alg.data.name, name, len);
} else {
param->attrs[i].nu32.attr.rta_len =
sizeof(param->attrs[i].nu32);
param->attrs[i].nu32.attr.rta_type = CRYPTOA_U32;
param->attrs[i].nu32.data.num =
simple_strtol(name, NULL, 0);
}
param->attrs[i].attr.rta_len = sizeof(param->attrs[i]);
param->attrs[i].attr.rta_type = CRYPTOA_ALG;
memcpy(param->attrs[i].data.name, name, len);
param->tb[i + 1] = &param->attrs[i].attr;
i++;

12
crypto/drbg.c
View File

@ -176,18 +176,18 @@ static const struct drbg_core drbg_cores[] = {
.blocklen_bytes = 48,
.cra_name = "hmac_sha384",
.backend_cra_name = "hmac(sha384)",
}, {
.flags = DRBG_HMAC | DRBG_STRENGTH256,
.statelen = 64, /* block length of cipher */
.blocklen_bytes = 64,
.cra_name = "hmac_sha512",
.backend_cra_name = "hmac(sha512)",
}, {
.flags = DRBG_HMAC | DRBG_STRENGTH256,
.statelen = 32, /* block length of cipher */
.blocklen_bytes = 32,
.cra_name = "hmac_sha256",
.backend_cra_name = "hmac(sha256)",
}, {
.flags = DRBG_HMAC | DRBG_STRENGTH256,
.statelen = 64, /* block length of cipher */
.blocklen_bytes = 64,
.cra_name = "hmac_sha512",
.backend_cra_name = "hmac(sha512)",
},
#endif /* CONFIG_CRYPTO_DRBG_HMAC */
};

49
crypto/ecdh.c
View File

@ -141,7 +141,7 @@ static struct kpp_alg ecdh_nist_p192 = {
.init = ecdh_nist_p192_init_tfm,
.base = {
.cra_name = "ecdh-nist-p192",
.cra_driver_name = "ecdh-generic",
.cra_driver_name = "ecdh-nist-p192-generic",
.cra_priority = 100,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct ecdh_ctx),
@ -166,7 +166,32 @@ static struct kpp_alg ecdh_nist_p256 = {
.init = ecdh_nist_p256_init_tfm,
.base = {
.cra_name = "ecdh-nist-p256",
.cra_driver_name = "ecdh-generic",
.cra_driver_name = "ecdh-nist-p256-generic",
.cra_priority = 100,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct ecdh_ctx),
},
};
static int ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
{
struct ecdh_ctx *ctx = ecdh_get_ctx(tfm);
ctx->curve_id = ECC_CURVE_NIST_P384;
ctx->ndigits = ECC_CURVE_NIST_P384_DIGITS;
return 0;
}
static struct kpp_alg ecdh_nist_p384 = {
.set_secret = ecdh_set_secret,
.generate_public_key = ecdh_compute_value,
.compute_shared_secret = ecdh_compute_value,
.max_size = ecdh_max_size,
.init = ecdh_nist_p384_init_tfm,
.base = {
.cra_name = "ecdh-nist-p384",
.cra_driver_name = "ecdh-nist-p384-generic",
.cra_priority = 100,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct ecdh_ctx),
@ -179,10 +204,27 @@ static int ecdh_init(void)
{
int ret;
/* NIST p192 will fail to register in FIPS mode */
ret = crypto_register_kpp(&ecdh_nist_p192);
ecdh_nist_p192_registered = ret == 0;
return crypto_register_kpp(&ecdh_nist_p256);
ret = crypto_register_kpp(&ecdh_nist_p256);
if (ret)
goto nist_p256_error;
ret = crypto_register_kpp(&ecdh_nist_p384);
if (ret)
goto nist_p384_error;
return 0;
nist_p384_error:
crypto_unregister_kpp(&ecdh_nist_p256);
nist_p256_error:
if (ecdh_nist_p192_registered)
crypto_unregister_kpp(&ecdh_nist_p192);
return ret;
}
static void ecdh_exit(void)
@ -190,6 +232,7 @@ static void ecdh_exit(void)
if (ecdh_nist_p192_registered)
crypto_unregister_kpp(&ecdh_nist_p192);
crypto_unregister_kpp(&ecdh_nist_p256);
crypto_unregister_kpp(&ecdh_nist_p384);
}
subsys_initcall(ecdh_init);

12
crypto/internal.h
View File

@ -29,6 +29,18 @@ struct crypto_larval {
u32 mask;
};
enum {
CRYPTOA_UNSPEC,
CRYPTOA_ALG,
CRYPTOA_TYPE,
__CRYPTOA_MAX,
};
#define CRYPTOA_MAX (__CRYPTOA_MAX - 1)
/* Maximum number of (rtattr) parameters for each template. */
#define CRYPTO_MAX_ATTRS 32
extern struct list_head crypto_alg_list;
extern struct rw_semaphore crypto_alg_sem;
extern struct blocking_notifier_head crypto_chain;

2
crypto/khazad.c
View File

@ -819,7 +819,7 @@ static void khazad_crypt(const u64 roundKey[KHAZAD_ROUNDS + 1],
T6[(int)(state >> 8) & 0xff] ^
T7[(int)(state ) & 0xff] ^
roundKey[r];
}
}
state = (T0[(int)(state >> 56) ] & 0xff00000000000000ULL) ^
(T1[(int)(state >> 48) & 0xff] & 0x00ff000000000000ULL) ^

18
crypto/shash.c
View File

@ -20,12 +20,24 @@
static const struct crypto_type crypto_shash_type;
int shash_no_setkey(struct crypto_shash *tfm, const u8 *key,
unsigned int keylen)
static int shash_no_setkey(struct crypto_shash *tfm, const u8 *key,
unsigned int keylen)
{
return -ENOSYS;
}
EXPORT_SYMBOL_GPL(shash_no_setkey);
/*
* Check whether an shash algorithm has a setkey function.
*
* For CFI compatibility, this must not be an inline function. This is because
* when CFI is enabled, modules won't get the same address for shash_no_setkey
* (if it were exported, which inlining would require) as the core kernel will.
*/
bool crypto_shash_alg_has_setkey(struct shash_alg *alg)
{
return alg->setkey != shash_no_setkey;
}
EXPORT_SYMBOL_GPL(crypto_shash_alg_has_setkey);
static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key,
unsigned int keylen)

24
crypto/sm2.c
View File

@ -79,10 +79,17 @@ static int sm2_ec_ctx_init(struct mpi_ec_ctx *ec)
goto free;
rc = -ENOMEM;
ec->Q = mpi_point_new(0);
if (!ec->Q)
goto free;
/* mpi_ec_setup_elliptic_curve */
ec->G = mpi_point_new(0);
if (!ec->G)
if (!ec->G) {
mpi_point_release(ec->Q);
goto free;
}
mpi_set(ec->G->x, x);
mpi_set(ec->G->y, y);
@ -91,6 +98,7 @@ static int sm2_ec_ctx_init(struct mpi_ec_ctx *ec)
rc = -EINVAL;
ec->n = mpi_scanval(ecp->n);
if (!ec->n) {
mpi_point_release(ec->Q);
mpi_point_release(ec->G);
goto free;
}
@ -386,27 +394,15 @@ static int sm2_set_pub_key(struct crypto_akcipher *tfm,
MPI a;
int rc;
ec->Q = mpi_point_new(0);
if (!ec->Q)
return -ENOMEM;
/* include the uncompressed flag '0x04' */
rc = -ENOMEM;
a = mpi_read_raw_data(key, keylen);
if (!a)
goto error;
return -ENOMEM;
mpi_normalize(a);
rc = sm2_ecc_os2ec(ec->Q, a);
mpi_free(a);
if (rc)
goto error;
return 0;
error:
mpi_point_release(ec->Q);
ec->Q = NULL;
return rc;
}

36
crypto/tcrypt.c
View File

@ -1847,10 +1847,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
ret += tcrypt_test("cts(cbc(aes))");
break;
case 39:
ret += tcrypt_test("xxhash64");
break;
case 40:
ret += tcrypt_test("rmd160");
break;
case 41:
ret += tcrypt_test("blake2s-256");
break;
case 42:
ret += tcrypt_test("blake2b-512");
break;
case 43:
ret += tcrypt_test("ecb(seed)");
break;
@ -2356,10 +2368,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
test_hash_speed("sha224", sec, generic_hash_speed_template);
if (mode > 300 && mode < 400) break;
fallthrough;
case 314:
test_hash_speed("xxhash64", sec, generic_hash_speed_template);
if (mode > 300 && mode < 400) break;
fallthrough;
case 315:
test_hash_speed("rmd160", sec, generic_hash_speed_template);
if (mode > 300 && mode < 400) break;
fallthrough;
case 316:
test_hash_speed("blake2s-256", sec, generic_hash_speed_template);
if (mode > 300 && mode < 400) break;
fallthrough;
case 317:
test_hash_speed("blake2b-512", sec, generic_hash_speed_template);
if (mode > 300 && mode < 400) break;
fallthrough;
case 318:
klen = 16;
test_hash_speed("ghash", sec, generic_hash_speed_template);
@ -2456,10 +2480,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
test_ahash_speed("sha224", sec, generic_hash_speed_template);
if (mode > 400 && mode < 500) break;
fallthrough;
case 414:
test_ahash_speed("xxhash64", sec, generic_hash_speed_template);
if (mode > 400 && mode < 500) break;
fallthrough;
case 415:
test_ahash_speed("rmd160", sec, generic_hash_speed_template);
if (mode > 400 && mode < 500) break;
fallthrough;
case 416:
test_ahash_speed("blake2s-256", sec, generic_hash_speed_template);
if (mode > 400 && mode < 500) break;
fallthrough;
case 417:
test_ahash_speed("blake2b-512", sec, generic_hash_speed_template);
if (mode > 400 && mode < 500) break;
fallthrough;
case 418:
test_ahash_speed("sha3-224", sec, generic_hash_speed_template);
if (mode > 400 && mode < 500) break;

10
crypto/testmgr.c
View File

@ -4899,21 +4899,25 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
#endif
#ifndef CONFIG_CRYPTO_FIPS
.alg = "ecdh-nist-p192",
.test = alg_test_kpp,
.fips_allowed = 1,
.suite = {
.kpp = __VECS(ecdh_p192_tv_template)
}
}, {
#endif
.alg = "ecdh-nist-p256",
.test = alg_test_kpp,
.fips_allowed = 1,
.suite = {
.kpp = __VECS(ecdh_p256_tv_template)
}
}, {
.alg = "ecdh-nist-p384",
.test = alg_test_kpp,
.fips_allowed = 1,
.suite = {
.kpp = __VECS(ecdh_p384_tv_template)
}
}, {
.alg = "ecdsa-nist-p192",
.test = alg_test_akcipher,

71
crypto/testmgr.h
View File

@ -2685,7 +2685,6 @@ static const struct kpp_testvec curve25519_tv_template[] = {
}
};
#ifndef CONFIG_CRYPTO_FIPS
static const struct kpp_testvec ecdh_p192_tv_template[] = {
{
.secret =
@ -2719,13 +2718,12 @@ static const struct kpp_testvec ecdh_p192_tv_template[] = {
"\xf4\x57\xcc\x4f\x1f\x4e\x31\xcc"
"\xe3\x40\x60\xc8\x06\x93\xc6\x2e"
"\x99\x80\x81\x28\xaf\xc5\x51\x74",
.secret_size = 32,
.secret_size = 30,
.b_public_size = 48,
.expected_a_public_size = 48,
.expected_ss_size = 24
}
};
#endif
static const struct kpp_testvec ecdh_p256_tv_template[] = {
{
@ -2766,7 +2764,7 @@ static const struct kpp_testvec ecdh_p256_tv_template[] = {
"\x9f\x4a\x38\xcc\xc0\x2c\x49\x2f"
"\xb1\x32\xbb\xaf\x22\x61\xda\xcb"
"\x6f\xdb\xa9\xaa\xfc\x77\x81\xf3",
.secret_size = 40,
.secret_size = 38,
.b_public_size = 64,
.expected_a_public_size = 64,
.expected_ss_size = 32
@ -2804,8 +2802,8 @@ static const struct kpp_testvec ecdh_p256_tv_template[] = {
"\x37\x08\xcc\x40\x5e\x7a\xfd\x6a"
"\x6a\x02\x6e\x41\x87\x68\x38\x77"
"\xfa\xa9\x44\x43\x2d\xef\x09\xdf",
.secret_size = 8,
.b_secret_size = 40,
.secret_size = 6,
.b_secret_size = 38,
.b_public_size = 64,
.expected_a_public_size = 64,
.expected_ss_size = 32,
@ -2813,6 +2811,67 @@ static const struct kpp_testvec ecdh_p256_tv_template[] = {
}
};
/*
* NIST P384 test vectors from RFC5903
*/
static const struct kpp_testvec ecdh_p384_tv_template[] = {
{
.secret =
#ifdef __LITTLE_ENDIAN
"\x02\x00" /* type */
"\x36\x00" /* len */
"\x30\x00" /* key_size */
#else
"\x00\x02" /* type */
"\x00\x36" /* len */
"\x00\x30" /* key_size */
#endif
"\x09\x9F\x3C\x70\x34\xD4\xA2\xC6"
"\x99\x88\x4D\x73\xA3\x75\xA6\x7F"
"\x76\x24\xEF\x7C\x6B\x3C\x0F\x16"
"\x06\x47\xB6\x74\x14\xDC\xE6\x55"
"\xE3\x5B\x53\x80\x41\xE6\x49\xEE"
"\x3F\xAE\xF8\x96\x78\x3A\xB1\x94",
.b_public =
"\xE5\x58\xDB\xEF\x53\xEE\xCD\xE3"
"\xD3\xFC\xCF\xC1\xAE\xA0\x8A\x89"
"\xA9\x87\x47\x5D\x12\xFD\x95\x0D"
"\x83\xCF\xA4\x17\x32\xBC\x50\x9D"
"\x0D\x1A\xC4\x3A\x03\x36\xDE\xF9"
"\x6F\xDA\x41\xD0\x77\x4A\x35\x71"
"\xDC\xFB\xEC\x7A\xAC\xF3\x19\x64"
"\x72\x16\x9E\x83\x84\x30\x36\x7F"
"\x66\xEE\xBE\x3C\x6E\x70\xC4\x16"
"\xDD\x5F\x0C\x68\x75\x9D\xD1\xFF"
"\xF8\x3F\xA4\x01\x42\x20\x9D\xFF"
"\x5E\xAA\xD9\x6D\xB9\xE6\x38\x6C",
.expected_a_public =
"\x66\x78\x42\xD7\xD1\x80\xAC\x2C"
"\xDE\x6F\x74\xF3\x75\x51\xF5\x57"
"\x55\xC7\x64\x5C\x20\xEF\x73\xE3"
"\x16\x34\xFE\x72\xB4\xC5\x5E\xE6"
"\xDE\x3A\xC8\x08\xAC\xB4\xBD\xB4"
"\xC8\x87\x32\xAE\xE9\x5F\x41\xAA"
"\x94\x82\xED\x1F\xC0\xEE\xB9\xCA"
"\xFC\x49\x84\x62\x5C\xCF\xC2\x3F"
"\x65\x03\x21\x49\xE0\xE1\x44\xAD"
"\xA0\x24\x18\x15\x35\xA0\xF3\x8E"
"\xEB\x9F\xCF\xF3\xC2\xC9\x47\xDA"
"\xE6\x9B\x4C\x63\x45\x73\xA8\x1C",
.expected_ss =
"\x11\x18\x73\x31\xC2\x79\x96\x2D"
"\x93\xD6\x04\x24\x3F\xD5\x92\xCB"
"\x9D\x0A\x92\x6F\x42\x2E\x47\x18"
"\x75\x21\x28\x7E\x71\x56\xC5\xC4"
"\xD6\x03\x13\x55\x69\xB9\xE9\xD0"
"\x9C\xF5\xD4\xA2\x70\xF5\x97\x46",
.secret_size = 54,
.b_public_size = 96,
.expected_a_public_size = 96,
.expected_ss_size = 48
}
};
/*
* MD4 test vectors from RFC1320
*/

40
crypto/wp512.c
View File

@ -1066,33 +1066,31 @@ static int wp512_final(struct shash_desc *desc, u8 *out)
{
struct wp512_ctx *wctx = shash_desc_ctx(desc);
int i;
u8 *buffer = wctx->buffer;
u8 *bitLength = wctx->bitLength;
int bufferBits = wctx->bufferBits;
int bufferPos = wctx->bufferPos;
u8 *buffer = wctx->buffer;
u8 *bitLength = wctx->bitLength;
int bufferBits = wctx->bufferBits;
int bufferPos = wctx->bufferPos;
__be64 *digest = (__be64 *)out;
buffer[bufferPos] |= 0x80U >> (bufferBits & 7);
bufferPos++;
if (bufferPos > WP512_BLOCK_SIZE - WP512_LENGTHBYTES) {
if (bufferPos < WP512_BLOCK_SIZE) {
memset(&buffer[bufferPos], 0, WP512_BLOCK_SIZE - bufferPos);
}
wp512_process_buffer(wctx);
bufferPos = 0;
}
if (bufferPos < WP512_BLOCK_SIZE - WP512_LENGTHBYTES) {
memset(&buffer[bufferPos], 0,
buffer[bufferPos] |= 0x80U >> (bufferBits & 7);
bufferPos++;
if (bufferPos > WP512_BLOCK_SIZE - WP512_LENGTHBYTES) {
if (bufferPos < WP512_BLOCK_SIZE)
memset(&buffer[bufferPos], 0, WP512_BLOCK_SIZE - bufferPos);
wp512_process_buffer(wctx);
bufferPos = 0;
}
if (bufferPos < WP512_BLOCK_SIZE - WP512_LENGTHBYTES)
memset(&buffer[bufferPos], 0,
(WP512_BLOCK_SIZE - WP512_LENGTHBYTES) - bufferPos);
}
bufferPos = WP512_BLOCK_SIZE - WP512_LENGTHBYTES;
memcpy(&buffer[WP512_BLOCK_SIZE - WP512_LENGTHBYTES],
bufferPos = WP512_BLOCK_SIZE - WP512_LENGTHBYTES;
memcpy(&buffer[WP512_BLOCK_SIZE - WP512_LENGTHBYTES],
bitLength, WP512_LENGTHBYTES);
wp512_process_buffer(wctx);
wp512_process_buffer(wctx);
for (i = 0; i < WP512_DIGEST_SIZE/8; i++)
digest[i] = cpu_to_be64(wctx->hash[i]);
wctx->bufferBits = bufferBits;
wctx->bufferPos = bufferPos;
wctx->bufferBits = bufferBits;
wctx->bufferPos = bufferPos;
return 0;
}

10
drivers/char/hw_random/Kconfig
View File

@ -165,17 +165,17 @@ config HW_RANDOM_IXP4XX
config HW_RANDOM_OMAP
tristate "OMAP Random Number Generator support"
depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU
depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU || ARCH_K3
default HW_RANDOM
help
This driver provides kernel-side support for the Random Number
This driver provides kernel-side support for the Random Number
Generator hardware found on OMAP16xx, OMAP2/3/4/5, AM33xx/AM43xx
multimedia processors, and Marvell Armada 7k/8k SoCs.
To compile this driver as a module, choose M here: the
module will be called omap-rng.
If unsure, say Y.
If unsure, say Y.
config HW_RANDOM_OMAP3_ROM
tristate "OMAP3 ROM Random Number Generator support"
@ -485,13 +485,13 @@ config HW_RANDOM_NPCM
depends on ARCH_NPCM || COMPILE_TEST
default HW_RANDOM
help
This driver provides support for the Random Number
This driver provides support for the Random Number
Generator hardware available in Nuvoton NPCM SoCs.
To compile this driver as a module, choose M here: the
module will be called npcm-rng.
If unsure, say Y.
If unsure, say Y.
config HW_RANDOM_KEYSTONE
depends on ARCH_KEYSTONE || COMPILE_TEST

2
drivers/char/hw_random/amd-rng.c
View File

@ -126,7 +126,7 @@ static struct hwrng amd_rng = {
static int __init mod_init(void)
{
int err = -ENODEV;
int err;
struct pci_dev *pdev = NULL;
const struct pci_device_id *ent;
u32 pmbase;

38
drivers/char/hw_random/core.c
View File

@ -319,11 +319,11 @@ static int enable_best_rng(void)
return ret;
}
static ssize_t hwrng_attr_current_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t len)
static ssize_t rng_current_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t len)
{
int err = -ENODEV;
int err;
struct hwrng *rng, *old_rng, *new_rng;
err = mutex_lock_interruptible(&rng_mutex);
@ -354,9 +354,9 @@ static ssize_t hwrng_attr_current_store(struct device *dev,
return err ? : len;
}
static ssize_t hwrng_attr_current_show(struct device *dev,
struct device_attribute *attr,
char *buf)
static ssize_t rng_current_show(struct device *dev,
struct device_attribute *attr,
char *buf)
{
ssize_t ret;
struct hwrng *rng;
@ -371,9 +371,9 @@ static ssize_t hwrng_attr_current_show(struct device *dev,
return ret;
}
static ssize_t hwrng_attr_available_show(struct device *dev,
struct device_attribute *attr,
char *buf)
static ssize_t rng_available_show(struct device *dev,
struct device_attribute *attr,
char *buf)
{
int err;
struct hwrng *rng;
@ -392,22 +392,16 @@ static ssize_t hwrng_attr_available_show(struct device *dev,
return strlen(buf);
}
static ssize_t hwrng_attr_selected_show(struct device *dev,
struct device_attribute *attr,
char *buf)
static ssize_t rng_selected_show(struct device *dev,
struct device_attribute *attr,
char *buf)
{
return sysfs_emit(buf, "%d\n", cur_rng_set_by_user);
}
static DEVICE_ATTR(rng_current, S_IRUGO | S_IWUSR,
hwrng_attr_current_show,
hwrng_attr_current_store);
static DEVICE_ATTR(rng_available, S_IRUGO,
hwrng_attr_available_show,
NULL);
static DEVICE_ATTR(rng_selected, S_IRUGO,
hwrng_attr_selected_show,
NULL);
static DEVICE_ATTR_RW(rng_current);
static DEVICE_ATTR_RO(rng_available);
static DEVICE_ATTR_RO(rng_selected);
static struct attribute *rng_dev_attrs[] = {
&dev_attr_rng_current.attr,

7
drivers/char/hw_random/exynos-trng.c
View File

@ -132,7 +132,7 @@ static int exynos_trng_probe(struct platform_device *pdev)
return PTR_ERR(trng->mem);
pm_runtime_enable(&pdev->dev);
ret = pm_runtime_get_sync(&pdev->dev);
ret = pm_runtime_resume_and_get(&pdev->dev);
if (ret < 0) {
dev_err(&pdev->dev, "Could not get runtime PM.\n");
goto err_pm_get;
@ -165,7 +165,7 @@ static int exynos_trng_probe(struct platform_device *pdev)
clk_disable_unprepare(trng->clk);
err_clock:
pm_runtime_put_sync(&pdev->dev);
pm_runtime_put_noidle(&pdev->dev);
err_pm_get:
pm_runtime_disable(&pdev->dev);
@ -196,10 +196,9 @@ static int __maybe_unused exynos_trng_resume(struct device *dev)
{
int ret;
ret = pm_runtime_get_sync(dev);
ret = pm_runtime_resume_and_get(dev);
if (ret < 0) {
dev_err(dev, "Could not get runtime PM.\n");
pm_runtime_put_noidle(dev);
return ret;
}

3
drivers/char/hw_random/ks-sa-rng.c
View File

@ -241,10 +241,9 @@ static int ks_sa_rng_probe(struct platform_device *pdev)
}
pm_runtime_enable(dev);
ret = pm_runtime_get_sync(dev);
ret = pm_runtime_resume_and_get(dev);
if (ret < 0) {
dev_err(dev, "Failed to enable SA power-domain\n");
pm_runtime_put_noidle(dev);
pm_runtime_disable(dev);
return ret;
}

6
drivers/char/hw_random/omap-rng.c
View File

@ -454,10 +454,9 @@ static int omap_rng_probe(struct platform_device *pdev)
}
pm_runtime_enable(&pdev->dev);
ret = pm_runtime_get_sync(&pdev->dev);
ret = pm_runtime_resume_and_get(&pdev->dev);
if (ret < 0) {
dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret);
pm_runtime_put_noidle(&pdev->dev);
goto err_ioremap;
}
@ -543,10 +542,9 @@ static int __maybe_unused omap_rng_resume(struct device *dev)
struct omap_rng_dev *priv = dev_get_drvdata(dev);
int ret;
ret = pm_runtime_get_sync(dev);
ret = pm_runtime_resume_and_get(dev);
if (ret < 0) {
dev_err(dev, "Failed to runtime_get device: %d\n", ret);
pm_runtime_put_noidle(dev);
return ret;
}

41
drivers/crypto/Kconfig
View File

@ -266,6 +266,27 @@ config CRYPTO_DEV_NIAGARA2
Group, which can perform encryption, decryption, hashing,
checksumming, and raw copies.
config CRYPTO_DEV_SL3516
tristate "Stormlink SL3516 crypto offloader"
depends on HAS_IOMEM
select CRYPTO_SKCIPHER
select CRYPTO_ENGINE
select CRYPTO_ECB
select CRYPTO_AES
select HW_RANDOM
depends on PM
help
This option allows you to have support for SL3516 crypto offloader.
config CRYPTO_DEV_SL3516_DEBUG
bool "Enable SL3516 stats"
depends on CRYPTO_DEV_SL3516
depends on DEBUG_FS
help
Say y to enable SL3516 debug stats.
This will create /sys/kernel/debug/sl3516/stats for displaying
the number of requests per algorithm and other internal stats.
config CRYPTO_DEV_HIFN_795X
tristate "Driver HIFN 795x crypto accelerator chips"
select CRYPTO_LIB_DES
@ -325,6 +346,11 @@ config CRYPTO_DEV_TALITOS2
config CRYPTO_DEV_IXP4XX
tristate "Driver for IXP4xx crypto hardware acceleration"
depends on ARCH_IXP4XX && IXP4XX_QMGR && IXP4XX_NPE
select CRYPTO_AES
select CRYPTO_DES
select CRYPTO_ECB
select CRYPTO_CBC
select CRYPTO_CTR
select CRYPTO_LIB_DES
select CRYPTO_AEAD
select CRYPTO_AUTHENC
@ -627,6 +653,12 @@ config CRYPTO_DEV_QCE_SHA
select CRYPTO_SHA1
select CRYPTO_SHA256
config CRYPTO_DEV_QCE_AEAD
bool
depends on CRYPTO_DEV_QCE
select CRYPTO_AUTHENC
select CRYPTO_LIB_DES
choice
prompt "Algorithms enabled for QCE acceleration"
default CRYPTO_DEV_QCE_ENABLE_ALL
@ -647,6 +679,7 @@ choice
bool "All supported algorithms"
select CRYPTO_DEV_QCE_SKCIPHER
select CRYPTO_DEV_QCE_SHA
select CRYPTO_DEV_QCE_AEAD
help
Enable all supported algorithms:
- AES (CBC, CTR, ECB, XTS)
@ -672,6 +705,14 @@ choice
- SHA1, HMAC-SHA1
- SHA256, HMAC-SHA256
config CRYPTO_DEV_QCE_ENABLE_AEAD
bool "AEAD algorithms only"
select CRYPTO_DEV_QCE_AEAD
help
Enable AEAD algorithms only:
- authenc()
- ccm(aes)
- rfc4309(ccm(aes))
endchoice
config CRYPTO_DEV_QCE_SW_MAX_LEN

1
drivers/crypto/Makefile
View File

@ -38,6 +38,7 @@ obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
obj-$(CONFIG_CRYPTO_DEV_SA2UL) += sa2ul.o
obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o
obj-$(CONFIG_CRYPTO_DEV_SL3516) += gemini/
obj-$(CONFIG_ARCH_STM32) += stm32/
obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/

2
drivers/crypto/cavium/cpt/cptpf_main.c
View File

@ -401,7 +401,7 @@ static void cpt_disable_all_cores(struct cpt_device *cpt)
cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0);
}
/**
/*
* Ensure all cores are disengaged from all groups by
* calling cpt_disable_all_cores() before calling this
* function.

10
drivers/crypto/cavium/cpt/cptvf_reqmanager.c
View File

@ -9,8 +9,8 @@
/**
* get_free_pending_entry - get free entry from pending queue
* @param pqinfo: pending_qinfo structure
* @param qno: queue number
* @q: pending queue
* @qlen: queue length
*/
static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
int qlen)
@ -244,11 +244,7 @@ static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
memcpy(ent, (void *)cmd, qinfo->cmd_size);
if (++queue->idx >= queue->qhead->size / 64) {
struct hlist_node *node;
hlist_for_each(node, &queue->chead) {
chunk = hlist_entry(node, struct command_chunk,
nextchunk);
hlist_for_each_entry(chunk, &queue->chead, nextchunk) {
if (chunk == queue->qhead) {
continue;
} else {

4
drivers/crypto/cavium/nitrox/nitrox_isr.c
View File

@ -307,6 +307,10 @@ int nitrox_register_interrupts(struct nitrox_device *ndev)
* Entry 192: NPS_CORE_INT_ACTIVE
*/
nr_vecs = pci_msix_vec_count(pdev);
if (nr_vecs < 0) {
dev_err(DEV(ndev), "Error in getting vec count %d\n", nr_vecs);
return nr_vecs;
}
/* Enable MSI-X */
ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);

21
drivers/crypto/cavium/nitrox/nitrox_main.c
View File

@ -35,7 +35,7 @@ static LIST_HEAD(ndevlist);
static DEFINE_MUTEX(devlist_lock);
static unsigned int num_devices;
/**
/*
* nitrox_pci_tbl - PCI Device ID Table
*/
static const struct pci_device_id nitrox_pci_tbl[] = {
@ -65,7 +65,7 @@ struct ucode {
u64 code[];
};
/**
/*
* write_to_ucd_unit - Write Firmware to NITROX UCD unit
*/
static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
@ -424,8 +424,7 @@ static int nitrox_probe(struct pci_dev *pdev,
err = nitrox_device_flr(pdev);
if (err) {
dev_err(&pdev->dev, "FLR failed\n");
pci_disable_device(pdev);
return err;
goto flr_fail;
}
if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
@ -434,16 +433,13 @@ static int nitrox_probe(struct pci_dev *pdev,
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
if (err) {
dev_err(&pdev->dev, "DMA configuration failed\n");
pci_disable_device(pdev);
return err;
goto flr_fail;
}
}
err = pci_request_mem_regions(pdev, nitrox_driver_name);
if (err) {
pci_disable_device(pdev);
return err;
}
if (err)
goto flr_fail;
pci_set_master(pdev);
ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
@ -479,7 +475,7 @@ static int nitrox_probe(struct pci_dev *pdev,
err = nitrox_pf_sw_init(ndev);
if (err)
goto ioremap_err;
goto pf_sw_fail;
err = nitrox_pf_hw_init(ndev);
if (err)
@ -509,12 +505,15 @@ static int nitrox_probe(struct pci_dev *pdev,
smp_mb__after_atomic();
pf_hw_fail:
nitrox_pf_sw_cleanup(ndev);
pf_sw_fail:
iounmap(ndev->bar_addr);
ioremap_err:
nitrox_remove_from_devlist(ndev);
kfree(ndev);
pci_set_drvdata(pdev, NULL);
ndev_fail:
pci_release_mem_regions(pdev);
flr_fail:
pci_disable_device(pdev);
return err;
}

4
drivers/crypto/cavium/nitrox/nitrox_mbx.c
View File

@ -8,7 +8,7 @@
#define RING_TO_VFNO(_x, _y) ((_x) / (_y))
/**
/*
* mbx_msg_type - Mailbox message types
*/
enum mbx_msg_type {
@ -18,7 +18,7 @@ enum mbx_msg_type {
MBX_MSG_TYPE_NACK,
};
/**
/*
* mbx_msg_opcode - Mailbox message opcodes
*/
enum mbx_msg_opcode {

16
drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
View File

@ -19,7 +19,7 @@
#define REQ_BACKLOG 2
#define REQ_POSTED 3
/**
/*
* Response codes from SE microcode
* 0x00 - Success
* Completion with no error
@ -159,7 +159,7 @@ static int dma_map_inbufs(struct nitrox_softreq *sr,
struct se_crypto_request *req)
{
struct device *dev = DEV(sr->ndev);
struct scatterlist *sg = req->src;
struct scatterlist *sg;
int i, nents, ret = 0;
nents = dma_map_sg(dev, req->src, sg_nents(req->src),
@ -279,6 +279,7 @@ static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
/**
* post_se_instr - Post SE instruction to Packet Input ring
* @sr: Request structure
* @cmdq: Command queue structure
*
* Returns 0 if successful or a negative error code,
* if no space in ring.
@ -369,9 +370,11 @@ static int nitrox_enqueue_request(struct nitrox_softreq *sr)
}
/**
* nitrox_se_request - Send request to SE core
* nitrox_process_se_request - Send request to SE core
* @ndev: NITROX device
* @req: Crypto request
* @callback: Completion callback
* @cb_arg: Completion callback arguments
*
* Returns 0 on success, or a negative error code.
*/
@ -526,9 +529,8 @@ static bool sr_completed(struct nitrox_softreq *sr)
}
/**
* process_request_list - process completed requests
* @ndev: N5 device
* @qno: queue to operate
* process_response_list - process completed requests
* @cmdq: Command queue structure
*
* Returns the number of responses processed.
*/
@ -578,7 +580,7 @@ static void process_response_list(struct nitrox_cmdq *cmdq)
}
}
/**
/*
* pkt_slc_resp_tasklet - post processing of SE responses
*/
void pkt_slc_resp_tasklet(unsigned long data)

2
drivers/crypto/cavium/nitrox/nitrox_skcipher.c
View File

@ -20,7 +20,7 @@ struct nitrox_cipher {
enum flexi_cipher value;
};
/**
/*
* supported cipher list
*/
static const struct nitrox_cipher flexi_cipher_table[] = {

2
drivers/crypto/ccp/ccp-dev.c
View File

@ -470,7 +470,7 @@ int ccp_cmd_queue_thread(void *data)
/**
* ccp_alloc_struct - allocate and initialize the ccp_device struct
*
* @dev: device struct of the CCP
* @sp: sp_device struct of the CCP
*/
struct ccp_device *ccp_alloc_struct(struct sp_device *sp)
{

3
drivers/crypto/ccp/ccp-dmaengine.c
View File

@ -307,8 +307,7 @@ static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc)
spin_lock_irqsave(&chan->lock, flags);
cookie = dma_cookie_assign(tx_desc);
list_del(&desc->entry);
list_add_tail(&desc->entry, &chan->pending);
list_move_tail(&desc->entry, &chan->pending);
spin_unlock_irqrestore(&chan->lock, flags);

4
drivers/crypto/ccp/sev-dev.c
View File

@ -43,6 +43,10 @@ static int psp_probe_timeout = 5;
module_param(psp_probe_timeout, int, 0644);
MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe");
MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */
MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */
MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */
static bool psp_dead;
static int psp_timeout;

6
drivers/crypto/ccp/sp-pci.c
View File

@ -222,7 +222,7 @@ static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (ret) {
dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n",
ret);
goto e_err;
goto free_irqs;
}
}
@ -230,10 +230,12 @@ static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
ret = sp_init(sp);
if (ret)
goto e_err;
goto free_irqs;
return 0;
free_irqs:
sp_free_irqs(sp);
e_err:
dev_notice(dev, "initialization failed\n");
return ret;

2
drivers/crypto/gemini/Makefile Normal file
View File

@ -0,0 +1,2 @@
obj-$(CONFIG_CRYPTO_DEV_SL3516) += sl3516-ce.o
sl3516-ce-y += sl3516-ce-core.o sl3516-ce-cipher.o sl3516-ce-rng.o

387
drivers/crypto/gemini/sl3516-ce-cipher.c Normal file
View File

@ -0,0 +1,387 @@
// SPDX-License-Identifier: GPL-2.0
/*
* sl3516-ce-cipher.c - hardware cryptographic offloader for Stormlink SL3516 SoC
*
* Copyright (C) 2021 Corentin LABBE <clabbe@baylibre.com>
*
* This file adds support for AES cipher with 128,192,256 bits keysize in
* ECB mode.
*/
#include <linux/crypto.h>
#include <linux/dma-mapping.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/pm_runtime.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/skcipher.h>
#include "sl3516-ce.h"
/* sl3516_ce_need_fallback - check if a request can be handled by the CE */
static bool sl3516_ce_need_fallback(struct skcipher_request *areq)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
struct sl3516_ce_dev *ce = op->ce;
struct scatterlist *in_sg = areq->src;
struct scatterlist *out_sg = areq->dst;
struct scatterlist *sg;
if (areq->cryptlen == 0 || areq->cryptlen % 16) {
ce->fallback_mod16++;
return true;
}
/*
* check if we have enough descriptors for TX
* Note: TX need one control desc for each SG
*/
if (sg_nents(areq->src) > MAXDESC / 2) {
ce->fallback_sg_count_tx++;
return true;
}
/* check if we have enough descriptors for RX */
if (sg_nents(areq->dst) > MAXDESC) {
ce->fallback_sg_count_rx++;
return true;
}
sg = areq->src;
while (sg) {
if ((sg->length % 16) != 0) {
ce->fallback_mod16++;
return true;
}
if ((sg_dma_len(sg) % 16) != 0) {
ce->fallback_mod16++;
return true;
}
if (!IS_ALIGNED(sg->offset, 16)) {
ce->fallback_align16++;
return true;
}
sg = sg_next(sg);
}
sg = areq->dst;
while (sg) {
if ((sg->length % 16) != 0) {
ce->fallback_mod16++;
return true;
}
if ((sg_dma_len(sg) % 16) != 0) {
ce->fallback_mod16++;
return true;
}
if (!IS_ALIGNED(sg->offset, 16)) {
ce->fallback_align16++;
return true;
}
sg = sg_next(sg);
}
/* need same numbers of SG (with same length) for source and destination */
in_sg = areq->src;
out_sg = areq->dst;
while (in_sg && out_sg) {
if (in_sg->length != out_sg->length) {
ce->fallback_not_same_len++;
return true;
}
in_sg = sg_next(in_sg);
out_sg = sg_next(out_sg);
}
if (in_sg || out_sg)
return true;
return false;
}
static int sl3516_ce_cipher_fallback(struct skcipher_request *areq)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
struct sl3516_ce_alg_template *algt;
int err;
algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher);
algt->stat_fb++;
skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
areq->base.complete, areq->base.data);
skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
areq->cryptlen, areq->iv);
if (rctx->op_dir == CE_DECRYPTION)
err = crypto_skcipher_decrypt(&rctx->fallback_req);
else
err = crypto_skcipher_encrypt(&rctx->fallback_req);
return err;
}
static int sl3516_ce_cipher(struct skcipher_request *areq)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
struct sl3516_ce_dev *ce = op->ce;
struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
struct sl3516_ce_alg_template *algt;
struct scatterlist *sg;
unsigned int todo, len;
struct pkt_control_ecb *ecb;
int nr_sgs = 0;
int nr_sgd = 0;
int err = 0;
int i;
algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher);
dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__,
crypto_tfm_alg_name(areq->base.tfm),
areq->cryptlen,
rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm),
op->keylen);
algt->stat_req++;
if (areq->src == areq->dst) {
nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src),
DMA_BIDIRECTIONAL);
if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) {
dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
err = -EINVAL;
goto theend;
}
nr_sgd = nr_sgs;
} else {
nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src),
DMA_TO_DEVICE);
if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) {
dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
err = -EINVAL;
goto theend;
}
nr_sgd = dma_map_sg(ce->dev, areq->dst, sg_nents(areq->dst),
DMA_FROM_DEVICE);
if (nr_sgd <= 0 || nr_sgd > MAXDESC) {
dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd);
err = -EINVAL;
goto theend_sgs;
}
}
len = areq->cryptlen;
i = 0;
sg = areq->src;
while (i < nr_sgs && sg && len) {
if (sg_dma_len(sg) == 0)
goto sgs_next;
rctx->t_src[i].addr = sg_dma_address(sg);
todo = min(len, sg_dma_len(sg));
rctx->t_src[i].len = todo;
dev_dbg(ce->dev, "%s total=%u SGS(%d %u off=%d) todo=%u\n", __func__,
areq->cryptlen, i, rctx->t_src[i].len, sg->offset, todo);
len -= todo;
i++;
sgs_next:
sg = sg_next(sg);
}
if (len > 0) {
dev_err(ce->dev, "remaining len %d/%u nr_sgs=%d\n", len, areq->cryptlen, nr_sgs);
err = -EINVAL;
goto theend_sgs;
}
len = areq->cryptlen;
i = 0;
sg = areq->dst;
while (i < nr_sgd && sg && len) {
if (sg_dma_len(sg) == 0)
goto sgd_next;
rctx->t_dst[i].addr = sg_dma_address(sg);
todo = min(len, sg_dma_len(sg));
rctx->t_dst[i].len = todo;
dev_dbg(ce->dev, "%s total=%u SGD(%d %u off=%d) todo=%u\n", __func__,
areq->cryptlen, i, rctx->t_dst[i].len, sg->offset, todo);
len -= todo;
i++;
sgd_next:
sg = sg_next(sg);
}
if (len > 0) {
dev_err(ce->dev, "remaining len %d\n", len);
err = -EINVAL;
goto theend_sgs;
}
switch (algt->mode) {
case ECB_AES:
rctx->pctrllen = sizeof(struct pkt_control_ecb);
ecb = (struct pkt_control_ecb *)ce->pctrl;
rctx->tqflag = TQ0_TYPE_CTRL;
rctx->tqflag |= TQ1_CIPHER;
ecb->control.op_mode = rctx->op_dir;
ecb->control.cipher_algorithm = ECB_AES;
ecb->cipher.header_len = 0;
ecb->cipher.algorithm_len = areq->cryptlen;
cpu_to_be32_array((__be32 *)ecb->key, (u32 *)op->key, op->keylen / 4);
rctx->h = &ecb->cipher;
rctx->tqflag |= TQ4_KEY0;
rctx->tqflag |= TQ5_KEY4;
rctx->tqflag |= TQ6_KEY6;
ecb->control.aesnk = op->keylen / 4;
break;
}
rctx->nr_sgs = nr_sgs;
rctx->nr_sgd = nr_sgd;
err = sl3516_ce_run_task(ce, rctx, crypto_tfm_alg_name(areq->base.tfm));
theend_sgs:
if (areq->src == areq->dst) {
dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src),
DMA_BIDIRECTIONAL);
} else {
dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src),
DMA_TO_DEVICE);
dma_unmap_sg(ce->dev, areq->dst, sg_nents(areq->dst),
DMA_FROM_DEVICE);
}
theend:
return err;
}
static int sl3516_ce_handle_cipher_request(struct crypto_engine *engine, void *areq)
{
int err;
struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
err = sl3516_ce_cipher(breq);
crypto_finalize_skcipher_request(engine, breq, err);
return 0;
}
int sl3516_ce_skdecrypt(struct skcipher_request *areq)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
struct crypto_engine *engine;
memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx));
rctx->op_dir = CE_DECRYPTION;
if (sl3516_ce_need_fallback(areq))
return sl3516_ce_cipher_fallback(areq);
engine = op->ce->engine;
return crypto_transfer_skcipher_request_to_engine(engine, areq);
}
int sl3516_ce_skencrypt(struct skcipher_request *areq)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
struct crypto_engine *engine;
memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx));
rctx->op_dir = CE_ENCRYPTION;
if (sl3516_ce_need_fallback(areq))
return sl3516_ce_cipher_fallback(areq);
engine = op->ce->engine;
return crypto_transfer_skcipher_request_to_engine(engine, areq);
}
int sl3516_ce_cipher_init(struct crypto_tfm *tfm)
{
struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
struct sl3516_ce_alg_template *algt;
const char *name = crypto_tfm_alg_name(tfm);
struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);
struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);
int err;
memset(op, 0, sizeof(struct sl3516_ce_cipher_tfm_ctx));
algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher);
op->ce = algt->ce;
op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(op->fallback_tfm)) {
dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
name, PTR_ERR(op->fallback_tfm));
return PTR_ERR(op->fallback_tfm);
}
sktfm->reqsize = sizeof(struct sl3516_ce_cipher_req_ctx) +
crypto_skcipher_reqsize(op->fallback_tfm);
dev_info(op->ce->dev, "Fallback for %s is %s\n",
crypto_tfm_alg_driver_name(&sktfm->base),
crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)));
op->enginectx.op.do_one_request = sl3516_ce_handle_cipher_request;
op->enginectx.op.prepare_request = NULL;
op->enginectx.op.unprepare_request = NULL;
err = pm_runtime_get_sync(op->ce->dev);
if (err < 0)
goto error_pm;
return 0;
error_pm:
pm_runtime_put_noidle(op->ce->dev);
crypto_free_skcipher(op->fallback_tfm);
return err;
}
void sl3516_ce_cipher_exit(struct crypto_tfm *tfm)
{
struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
kfree_sensitive(op->key);
crypto_free_skcipher(op->fallback_tfm);
pm_runtime_put_sync_suspend(op->ce->dev);
}
int sl3516_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int keylen)
{
struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
struct sl3516_ce_dev *ce = op->ce;
switch (keylen) {
case 128 / 8:
break;
case 192 / 8:
break;
case 256 / 8:
break;
default:
dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen);
return -EINVAL;
}
kfree_sensitive(op->key);
op->keylen = keylen;
op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
if (!op->key)
return -ENOMEM;
crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);
crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
}

535
drivers/crypto/gemini/sl3516-ce-core.c Normal file
View File

@ -0,0 +1,535 @@
// SPDX-License-Identifier: GPL-2.0
/*
* sl3516-ce-core.c - hardware cryptographic offloader for Stormlink SL3516 SoC
*
* Copyright (C) 2021 Corentin Labbe <clabbe@baylibre.com>
*
* Core file which registers crypto algorithms supported by the CryptoEngine
*/
#include <linux/clk.h>
#include <linux/crypto.h>
#include <linux/debugfs.h>
#include <linux/dev_printk.h>
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/irq.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
#include <crypto/internal/rng.h>
#include <crypto/internal/skcipher.h>
#include "sl3516-ce.h"
static int sl3516_ce_desc_init(struct sl3516_ce_dev *ce)
{
const size_t sz = sizeof(struct descriptor) * MAXDESC;
int i;
ce->tx = dma_alloc_coherent(ce->dev, sz, &ce->dtx, GFP_KERNEL);
if (!ce->tx)
return -ENOMEM;
ce->rx = dma_alloc_coherent(ce->dev, sz, &ce->drx, GFP_KERNEL);
if (!ce->rx)
goto err_rx;
for (i = 0; i < MAXDESC; i++) {
ce->tx[i].frame_ctrl.bits.own = CE_CPU;
ce->tx[i].next_desc.next_descriptor = ce->dtx + (i + 1) * sizeof(struct descriptor);
}
ce->tx[MAXDESC - 1].next_desc.next_descriptor = ce->dtx;
for (i = 0; i < MAXDESC; i++) {
ce->rx[i].frame_ctrl.bits.own = CE_CPU;
ce->rx[i].next_desc.next_descriptor = ce->drx + (i + 1) * sizeof(struct descriptor);
}
ce->rx[MAXDESC - 1].next_desc.next_descriptor = ce->drx;
ce->pctrl = dma_alloc_coherent(ce->dev, sizeof(struct pkt_control_ecb),
&ce->dctrl, GFP_KERNEL);
if (!ce->pctrl)
goto err_pctrl;
return 0;
err_pctrl:
dma_free_coherent(ce->dev, sz, ce->rx, ce->drx);
err_rx:
dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx);
return -ENOMEM;
}
static void sl3516_ce_free_descs(struct sl3516_ce_dev *ce)
{
const size_t sz = sizeof(struct descriptor) * MAXDESC;
dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx);
dma_free_coherent(ce->dev, sz, ce->rx, ce->drx);
dma_free_coherent(ce->dev, sizeof(struct pkt_control_ecb), ce->pctrl,
ce->dctrl);
}
static void start_dma_tx(struct sl3516_ce_dev *ce)
{
u32 v;
v = TXDMA_CTRL_START | TXDMA_CTRL_CHAIN_MODE | TXDMA_CTRL_CONTINUE | \
TXDMA_CTRL_INT_FAIL | TXDMA_CTRL_INT_PERR | TXDMA_CTRL_BURST_UNK;
writel(v, ce->base + IPSEC_TXDMA_CTRL);
}
static void start_dma_rx(struct sl3516_ce_dev *ce)
{
u32 v;
v = RXDMA_CTRL_START | RXDMA_CTRL_CHAIN_MODE | RXDMA_CTRL_CONTINUE | \
RXDMA_CTRL_BURST_UNK | RXDMA_CTRL_INT_FINISH | \
RXDMA_CTRL_INT_FAIL | RXDMA_CTRL_INT_PERR | \
RXDMA_CTRL_INT_EOD | RXDMA_CTRL_INT_EOF;
writel(v, ce->base + IPSEC_RXDMA_CTRL);
}
static struct descriptor *get_desc_tx(struct sl3516_ce_dev *ce)
{
struct descriptor *dd;
dd = &ce->tx[ce->ctx];
ce->ctx++;
if (ce->ctx >= MAXDESC)
ce->ctx = 0;
return dd;
}
static struct descriptor *get_desc_rx(struct sl3516_ce_dev *ce)
{
struct descriptor *rdd;
rdd = &ce->rx[ce->crx];
ce->crx++;
if (ce->crx >= MAXDESC)
ce->crx = 0;
return rdd;
}
int sl3516_ce_run_task(struct sl3516_ce_dev *ce, struct sl3516_ce_cipher_req_ctx *rctx,
const char *name)
{
struct descriptor *dd, *rdd = NULL;
u32 v;
int i, err = 0;
ce->stat_req++;
reinit_completion(&ce->complete);
ce->status = 0;
for (i = 0; i < rctx->nr_sgd; i++) {
dev_dbg(ce->dev, "%s handle DST SG %d/%d len=%d\n", __func__,
i, rctx->nr_sgd, rctx->t_dst[i].len);
rdd = get_desc_rx(ce);
rdd->buf_adr = rctx->t_dst[i].addr;
rdd->frame_ctrl.bits.buffer_size = rctx->t_dst[i].len;
rdd->frame_ctrl.bits.own = CE_DMA;
}
rdd->next_desc.bits.eofie = 1;
for (i = 0; i < rctx->nr_sgs; i++) {
dev_dbg(ce->dev, "%s handle SRC SG %d/%d len=%d\n", __func__,
i, rctx->nr_sgs, rctx->t_src[i].len);
rctx->h->algorithm_len = rctx->t_src[i].len;
dd = get_desc_tx(ce);
dd->frame_ctrl.raw = 0;
dd->flag_status.raw = 0;
dd->frame_ctrl.bits.buffer_size = rctx->pctrllen;
dd->buf_adr = ce->dctrl;
dd->flag_status.tx_flag.tqflag = rctx->tqflag;
dd->next_desc.bits.eofie = 0;
dd->next_desc.bits.dec = 0;
dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST;
dd->frame_ctrl.bits.own = CE_DMA;
dd = get_desc_tx(ce);
dd->frame_ctrl.raw = 0;
dd->flag_status.raw = 0;
dd->frame_ctrl.bits.buffer_size = rctx->t_src[i].len;
dd->buf_adr = rctx->t_src[i].addr;
dd->flag_status.tx_flag.tqflag = 0;
dd->next_desc.bits.eofie = 0;
dd->next_desc.bits.dec = 0;
dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST;
dd->frame_ctrl.bits.own = CE_DMA;
start_dma_tx(ce);
start_dma_rx(ce);
}
wait_for_completion_interruptible_timeout(&ce->complete,
msecs_to_jiffies(5000));
if (ce->status == 0) {
dev_err(ce->dev, "DMA timeout for %s\n", name);
err = -EFAULT;
}
v = readl(ce->base + IPSEC_STATUS_REG);
if (v & 0xFFF) {
dev_err(ce->dev, "IPSEC_STATUS_REG %x\n", v);
err = -EFAULT;
}
return err;
}
static irqreturn_t ce_irq_handler(int irq, void *data)
{
struct sl3516_ce_dev *ce = (struct sl3516_ce_dev *)data;
u32 v;
ce->stat_irq++;
v = readl(ce->base + IPSEC_DMA_STATUS);
writel(v, ce->base + IPSEC_DMA_STATUS);
if (v & DMA_STATUS_TS_DERR)
dev_err(ce->dev, "AHB bus Error While Tx !!!\n");
if (v & DMA_STATUS_TS_PERR)
dev_err(ce->dev, "Tx Descriptor Protocol Error !!!\n");
if (v & DMA_STATUS_RS_DERR)
dev_err(ce->dev, "AHB bus Error While Rx !!!\n");
if (v & DMA_STATUS_RS_PERR)
dev_err(ce->dev, "Rx Descriptor Protocol Error !!!\n");
if (v & DMA_STATUS_TS_EOFI)
ce->stat_irq_tx++;
if (v & DMA_STATUS_RS_EOFI) {
ce->status = 1;
complete(&ce->complete);
ce->stat_irq_rx++;
return IRQ_HANDLED;
}
return IRQ_HANDLED;
}
static struct sl3516_ce_alg_template ce_algs[] = {
{
.type = CRYPTO_ALG_TYPE_SKCIPHER,
.mode = ECB_AES,
.alg.skcipher = {
.base = {
.cra_name = "ecb(aes)",
.cra_driver_name = "ecb-aes-sl3516",
.cra_priority = 400,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
.cra_ctxsize = sizeof(struct sl3516_ce_cipher_tfm_ctx),
.cra_module = THIS_MODULE,
.cra_alignmask = 0xf,
.cra_init = sl3516_ce_cipher_init,
.cra_exit = sl3516_ce_cipher_exit,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = sl3516_ce_aes_setkey,
.encrypt = sl3516_ce_skencrypt,
.decrypt = sl3516_ce_skdecrypt,
}
},
};
#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG
static int sl3516_ce_debugfs_show(struct seq_file *seq, void *v)
{
struct sl3516_ce_dev *ce = seq->private;
unsigned int i;
seq_printf(seq, "HWRNG %lu %lu\n",
ce->hwrng_stat_req, ce->hwrng_stat_bytes);
seq_printf(seq, "IRQ %lu\n", ce->stat_irq);
seq_printf(seq, "IRQ TX %lu\n", ce->stat_irq_tx);
seq_printf(seq, "IRQ RX %lu\n", ce->stat_irq_rx);
seq_printf(seq, "nreq %lu\n", ce->stat_req);
seq_printf(seq, "fallback SG count TX %lu\n", ce->fallback_sg_count_tx);
seq_printf(seq, "fallback SG count RX %lu\n", ce->fallback_sg_count_rx);
seq_printf(seq, "fallback modulo16 %lu\n", ce->fallback_mod16);
seq_printf(seq, "fallback align16 %lu\n", ce->fallback_align16);
seq_printf(seq, "fallback not same len %lu\n", ce->fallback_not_same_len);
for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
if (!ce_algs[i].ce)
continue;
switch (ce_algs[i].type) {
case CRYPTO_ALG_TYPE_SKCIPHER:
seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
ce_algs[i].alg.skcipher.base.cra_driver_name,
ce_algs[i].alg.skcipher.base.cra_name,
ce_algs[i].stat_req, ce_algs[i].stat_fb);
break;
}
}
return 0;
}
DEFINE_SHOW_ATTRIBUTE(sl3516_ce_debugfs);
#endif
static int sl3516_ce_register_algs(struct sl3516_ce_dev *ce)
{
int err;
unsigned int i;
for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
ce_algs[i].ce = ce;
switch (ce_algs[i].type) {
case CRYPTO_ALG_TYPE_SKCIPHER:
dev_info(ce->dev, "DEBUG: Register %s\n",
ce_algs[i].alg.skcipher.base.cra_name);
err = crypto_register_skcipher(&ce_algs[i].alg.skcipher);
if (err) {
dev_err(ce->dev, "Fail to register %s\n",
ce_algs[i].alg.skcipher.base.cra_name);
ce_algs[i].ce = NULL;
return err;
}
break;
default:
ce_algs[i].ce = NULL;
dev_err(ce->dev, "ERROR: tried to register an unknown algo\n");
}
}
return 0;
}
static void sl3516_ce_unregister_algs(struct sl3516_ce_dev *ce)
{
unsigned int i;
for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
if (!ce_algs[i].ce)
continue;
switch (ce_algs[i].type) {
case CRYPTO_ALG_TYPE_SKCIPHER:
dev_info(ce->dev, "Unregister %d %s\n", i,
ce_algs[i].alg.skcipher.base.cra_name);
crypto_unregister_skcipher(&ce_algs[i].alg.skcipher);
break;
}
}
}
static void sl3516_ce_start(struct sl3516_ce_dev *ce)
{
ce->ctx = 0;
ce->crx = 0;
writel(ce->dtx, ce->base + IPSEC_TXDMA_CURR_DESC);
writel(ce->drx, ce->base + IPSEC_RXDMA_CURR_DESC);
writel(0, ce->base + IPSEC_DMA_STATUS);
}
/*
* Power management strategy: The device is suspended unless a TFM exists for
* one of the algorithms proposed by this driver.
*/
static int sl3516_ce_pm_suspend(struct device *dev)
{
struct sl3516_ce_dev *ce = dev_get_drvdata(dev);
reset_control_assert(ce->reset);
clk_disable_unprepare(ce->clks);
return 0;
}
static int sl3516_ce_pm_resume(struct device *dev)
{
struct sl3516_ce_dev *ce = dev_get_drvdata(dev);
int err;
err = clk_prepare_enable(ce->clks);
if (err) {
dev_err(ce->dev, "Cannot prepare_enable\n");
goto error;
}
err = reset_control_deassert(ce->reset);
if (err) {
dev_err(ce->dev, "Cannot deassert reset control\n");
goto error;
}
sl3516_ce_start(ce);
return 0;
error:
sl3516_ce_pm_suspend(dev);
return err;
}
static const struct dev_pm_ops sl3516_ce_pm_ops = {
SET_RUNTIME_PM_OPS(sl3516_ce_pm_suspend, sl3516_ce_pm_resume, NULL)
};
static int sl3516_ce_pm_init(struct sl3516_ce_dev *ce)
{
int err;
pm_runtime_use_autosuspend(ce->dev);
pm_runtime_set_autosuspend_delay(ce->dev, 2000);
err = pm_runtime_set_suspended(ce->dev);
if (err)
return err;
pm_runtime_enable(ce->dev);
return err;
}
static void sl3516_ce_pm_exit(struct sl3516_ce_dev *ce)
{
pm_runtime_disable(ce->dev);
}
static int sl3516_ce_probe(struct platform_device *pdev)
{
struct sl3516_ce_dev *ce;
int err, irq;
u32 v;
ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL);
if (!ce)
return -ENOMEM;
ce->dev = &pdev->dev;
platform_set_drvdata(pdev, ce);
ce->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(ce->base))
return PTR_ERR(ce->base);
irq = platform_get_irq(pdev, 0);
if (irq < 0)
return irq;
err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0, "crypto", ce);
if (err) {
dev_err(ce->dev, "Cannot request Crypto Engine IRQ (err=%d)\n", err);
return err;
}
ce->reset = devm_reset_control_get(&pdev->dev, NULL);
if (IS_ERR(ce->reset))
return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset),
"No reset control found\n");
ce->clks = devm_clk_get(ce->dev, NULL);
if (IS_ERR(ce->clks)) {
err = PTR_ERR(ce->clks);
dev_err(ce->dev, "Cannot get clock err=%d\n", err);
return err;
}
err = sl3516_ce_desc_init(ce);
if (err)
return err;
err = sl3516_ce_pm_init(ce);
if (err)
goto error_pm;
init_completion(&ce->complete);
ce->engine = crypto_engine_alloc_init(ce->dev, true);
if (!ce->engine) {
dev_err(ce->dev, "Cannot allocate engine\n");
err = -ENOMEM;
goto error_engine;
}
err = crypto_engine_start(ce->engine);
if (err) {
dev_err(ce->dev, "Cannot start engine\n");
goto error_engine;
}
err = sl3516_ce_register_algs(ce);
if (err)
goto error_alg;
err = sl3516_ce_rng_register(ce);
if (err)
goto error_rng;
err = pm_runtime_resume_and_get(ce->dev);
if (err < 0)
goto error_pmuse;
v = readl(ce->base + IPSEC_ID);
dev_info(ce->dev, "SL3516 dev %lx rev %lx\n",
v & GENMASK(31, 4),
v & GENMASK(3, 0));
v = readl(ce->base + IPSEC_DMA_DEVICE_ID);
dev_info(ce->dev, "SL3516 DMA dev %lx rev %lx\n",
v & GENMASK(15, 4),
v & GENMASK(3, 0));
pm_runtime_put_sync(ce->dev);
#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG
/* Ignore error of debugfs */
ce->dbgfs_dir = debugfs_create_dir("sl3516", NULL);
ce->dbgfs_stats = debugfs_create_file("stats", 0444,
ce->dbgfs_dir, ce,
&sl3516_ce_debugfs_fops);
#endif
return 0;
error_pmuse:
sl3516_ce_rng_unregister(ce);
error_rng:
sl3516_ce_unregister_algs(ce);
error_alg:
crypto_engine_exit(ce->engine);
error_engine:
sl3516_ce_pm_exit(ce);
error_pm:
sl3516_ce_free_descs(ce);
return err;
}
static int sl3516_ce_remove(struct platform_device *pdev)
{
struct sl3516_ce_dev *ce = platform_get_drvdata(pdev);
sl3516_ce_rng_unregister(ce);
sl3516_ce_unregister_algs(ce);
crypto_engine_exit(ce->engine);
sl3516_ce_pm_exit(ce);
sl3516_ce_free_descs(ce);
#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG
debugfs_remove_recursive(ce->dbgfs_dir);
#endif
return 0;
}
static const struct of_device_id sl3516_ce_crypto_of_match_table[] = {
{ .compatible = "cortina,sl3516-crypto"},
{}
};
MODULE_DEVICE_TABLE(of, sl3516_ce_crypto_of_match_table);
static struct platform_driver sl3516_ce_driver = {
.probe = sl3516_ce_probe,
.remove = sl3516_ce_remove,
.driver = {
.name = "sl3516-crypto",
.pm = &sl3516_ce_pm_ops,
.of_match_table = sl3516_ce_crypto_of_match_table,
},
};
module_platform_driver(sl3516_ce_driver);
MODULE_DESCRIPTION("SL3516 cryptographic offloader");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Corentin Labbe <clabbe@baylibre.com>");

61
drivers/crypto/gemini/sl3516-ce-rng.c Normal file
View File

@ -0,0 +1,61 @@
// SPDX-License-Identifier: GPL-2.0
/*
* sl3516-ce-rng.c - hardware cryptographic offloader for SL3516 SoC.
*
* Copyright (C) 2021 Corentin Labbe <clabbe@baylibre.com>
*
* This file handle the RNG found in the SL3516 crypto engine
*/
#include "sl3516-ce.h"
#include <linux/pm_runtime.h>
#include <linux/hw_random.h>
static int sl3516_ce_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
{
struct sl3516_ce_dev *ce;
u32 *data = buf;
size_t read = 0;
int err;
ce = container_of(rng, struct sl3516_ce_dev, trng);
#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG
ce->hwrng_stat_req++;
ce->hwrng_stat_bytes += max;
#endif
err = pm_runtime_get_sync(ce->dev);
if (err < 0) {
pm_runtime_put_noidle(ce->dev);
return err;
}
while (read < max) {
*data = readl(ce->base + IPSEC_RAND_NUM_REG);
data++;
read += 4;
}
pm_runtime_put(ce->dev);
return read;
}
int sl3516_ce_rng_register(struct sl3516_ce_dev *ce)
{
int ret;
ce->trng.name = "SL3516 Crypto Engine RNG";
ce->trng.read = sl3516_ce_rng_read;
ce->trng.quality = 700;
ret = hwrng_register(&ce->trng);
if (ret)
dev_err(ce->dev, "Fail to register the RNG\n");
return ret;
}
void sl3516_ce_rng_unregister(struct sl3516_ce_dev *ce)
{
hwrng_unregister(&ce->trng);
}

347
drivers/crypto/gemini/sl3516-ce.h Normal file
View File

@ -0,0 +1,347 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* sl3516-ce.h - hardware cryptographic offloader for cortina/gemini SoC
*
* Copyright (C) 2021 Corentin LABBE <clabbe@baylibre.com>
*
* General notes on this driver:
* Called either Crypto Acceleration Engine Module, Security Acceleration Engine
* or IPSEC module in the datasheet, it will be called Crypto Engine for short
* in this driver.
* The CE was designed to handle IPSEC and wifi(TKIP WEP) protocol.
* It can handle AES, DES, 3DES, MD5, WEP, TKIP, SHA1, HMAC(MD5), HMAC(SHA1),
* Michael cipher/digest suites.
* It acts the same as a network hw, with both RX and TX chained descriptors.
*/
#include <crypto/aes.h>
#include <crypto/engine.h>
#include <crypto/scatterwalk.h>
#include <crypto/skcipher.h>
#include <linux/crypto.h>
#include <linux/debugfs.h>
#include <linux/hw_random.h>
#define TQ0_TYPE_DATA 0
#define TQ0_TYPE_CTRL BIT(0)
#define TQ1_CIPHER BIT(1)
#define TQ2_AUTH BIT(2)
#define TQ3_IV BIT(3)
#define TQ4_KEY0 BIT(4)
#define TQ5_KEY4 BIT(5)
#define TQ6_KEY6 BIT(6)
#define TQ7_AKEY0 BIT(7)
#define TQ8_AKEY2 BIT(8)
#define TQ9_AKEY2 BIT(9)
#define ECB_AES 0x2
#define DESC_LAST 0x01
#define DESC_FIRST 0x02
#define IPSEC_ID 0x0000
#define IPSEC_STATUS_REG 0x00a8
#define IPSEC_RAND_NUM_REG 0x00ac
#define IPSEC_DMA_DEVICE_ID 0xff00
#define IPSEC_DMA_STATUS 0xff04
#define IPSEC_TXDMA_CTRL 0xff08
#define IPSEC_TXDMA_FIRST_DESC 0xff0c
#define IPSEC_TXDMA_CURR_DESC 0xff10
#define IPSEC_RXDMA_CTRL 0xff14
#define IPSEC_RXDMA_FIRST_DESC 0xff18
#define IPSEC_RXDMA_CURR_DESC 0xff1c
#define IPSEC_TXDMA_BUF_ADDR 0xff28
#define IPSEC_RXDMA_BUF_ADDR 0xff38
#define IPSEC_RXDMA_BUF_SIZE 0xff30
#define CE_ENCRYPTION 0x01
#define CE_DECRYPTION 0x03
#define MAXDESC 6
#define DMA_STATUS_RS_EOFI BIT(22)
#define DMA_STATUS_RS_PERR BIT(24)
#define DMA_STATUS_RS_DERR BIT(25)
#define DMA_STATUS_TS_EOFI BIT(27)
#define DMA_STATUS_TS_PERR BIT(29)
#define DMA_STATUS_TS_DERR BIT(30)
#define TXDMA_CTRL_START BIT(31)
#define TXDMA_CTRL_CONTINUE BIT(30)
#define TXDMA_CTRL_CHAIN_MODE BIT(29)
/* the burst value is not documented in the datasheet */
#define TXDMA_CTRL_BURST_UNK BIT(22)
#define TXDMA_CTRL_INT_FAIL BIT(17)
#define TXDMA_CTRL_INT_PERR BIT(16)
#define RXDMA_CTRL_START BIT(31)
#define RXDMA_CTRL_CONTINUE BIT(30)
#define RXDMA_CTRL_CHAIN_MODE BIT(29)
/* the burst value is not documented in the datasheet */
#define RXDMA_CTRL_BURST_UNK BIT(22)
#define RXDMA_CTRL_INT_FINISH BIT(18)
#define RXDMA_CTRL_INT_FAIL BIT(17)
#define RXDMA_CTRL_INT_PERR BIT(16)
#define RXDMA_CTRL_INT_EOD BIT(15)
#define RXDMA_CTRL_INT_EOF BIT(14)
#define CE_CPU 0
#define CE_DMA 1
/*
* struct sl3516_ce_descriptor - descriptor for CE operations
* @frame_ctrl: Information for the current descriptor
* @flag_status: For send packet, describe flag of operations.
* @buf_adr: pointer to a send/recv buffer for data packet
* @next_desc: control linking to other descriptors
*/
struct descriptor {
union {
u32 raw;
/*
* struct desc_frame_ctrl - Information for the current descriptor
* @buffer_size: the size of buffer at buf_adr
* @desc_count: Upon completion of a DMA operation, DMA
* write the number of descriptors used
* for the current frame
* @checksum: unknown
* @authcomp: unknown
* @perr: Protocol error during processing this descriptor
* @derr: Data error during processing this descriptor
* @own: 0 if owned by CPU, 1 for DMA
*/
struct desc_frame_ctrl {
u32 buffer_size :16;
u32 desc_count :6;
u32 checksum :6;
u32 authcomp :1;
u32 perr :1;
u32 derr :1;
u32 own :1;
} bits;
} frame_ctrl;
union {
u32 raw;
/*
* struct desc_flag_status - flag for this descriptor
* @tqflag: list of flag describing the type of operation
* to be performed.
*/
struct desc_tx_flag_status {
u32 tqflag :10;
u32 unused :22;
} tx_flag;
} flag_status;
u32 buf_adr;
union {
u32 next_descriptor;
/*
* struct desc_next - describe chaining of descriptors
* @sof_eof: does the descriptor is first (0x11),
* the last (0x01), middle of a chan (0x00)
* or the only one (0x11)
* @dec: AHB bus address increase (0), decrease (1)
* @eofie: End of frame interrupt enable
* @ndar: Next descriptor address
*/
struct desc_next {
u32 sof_eof :2;
u32 dec :1;
u32 eofie :1;
u32 ndar :28;
} bits;
} next_desc;
};
/*
* struct control - The value of this register is used to set the
* operation mode of the IPSec Module.
* @process_id: Used to identify the process. The number will be copied
* to the descriptor status of the received packet.
* @auth_check_len: Number of 32-bit words to be checked or appended by the
* authentication module
* @auth_algorithm:
* @auth_mode: 0:append 1:Check Authentication Result
* @fcs_stream_copy: 0:enable 1:disable authentication stream copy
* @mix_key_sel: 0:use rCipherKey0-3 1:use Key Mixer
* @aesnk: AES Key Size
* @cipher_algorithm: choice of CBC/ECE and AES/DES/3DES
* @op_mode: Operation Mode for the IPSec Module
*/
struct pkt_control_header {
u32 process_id :8;
u32 auth_check_len :3;
u32 un1 :1;
u32 auth_algorithm :3;
u32 auth_mode :1;
u32 fcs_stream_copy :1;
u32 un2 :2;
u32 mix_key_sel :1;
u32 aesnk :4;
u32 cipher_algorithm :3;
u32 un3 :1;
u32 op_mode :4;
};
struct pkt_control_cipher {
u32 algorithm_len :16;
u32 header_len :16;
};
/*
* struct pkt_control_ecb - control packet for ECB
*/
struct pkt_control_ecb {
struct pkt_control_header control;
struct pkt_control_cipher cipher;
unsigned char key[AES_MAX_KEY_SIZE];
};
/*
* struct sl3516_ce_dev - main container for all this driver information
* @base: base address
* @clks: clocks used
* @reset: pointer to reset controller
* @dev: the platform device
* @engine: ptr to the crypto/crypto_engine
* @complete: completion for the current task on this flow
* @status: set to 1 by interrupt if task is done
* @dtx: base DMA address for TX descriptors
* @tx base address of TX descriptors
* @drx: base DMA address for RX descriptors
* @rx base address of RX descriptors
* @ctx current used TX descriptor
* @crx current used RX descriptor
* @trng hw_random structure for RNG
* @hwrng_stat_req number of HWRNG requests
* @hwrng_stat_bytes total number of bytes generated by RNG
* @stat_irq number of IRQ handled by CE
* @stat_irq_tx number of TX IRQ handled by CE
* @stat_irq_rx number of RX IRQ handled by CE
* @stat_req number of requests handled by CE
* @fallbak_sg_count_tx number of fallback due to destination SG count
* @fallbak_sg_count_rx number of fallback due to source SG count
* @fallbak_not_same_len number of fallback due to difference in SG length
* @dbgfs_dir: Debugfs dentry for statistic directory
* @dbgfs_stats: Debugfs dentry for statistic counters
*/
struct sl3516_ce_dev {
void __iomem *base;
struct clk *clks;
struct reset_control *reset;
struct device *dev;
struct crypto_engine *engine;
struct completion complete;
int status;
dma_addr_t dtx;
struct descriptor *tx;
dma_addr_t drx;
struct descriptor *rx;
int ctx;
int crx;
struct hwrng trng;
unsigned long hwrng_stat_req;
unsigned long hwrng_stat_bytes;
unsigned long stat_irq;
unsigned long stat_irq_tx;
unsigned long stat_irq_rx;
unsigned long stat_req;
unsigned long fallback_sg_count_tx;
unsigned long fallback_sg_count_rx;
unsigned long fallback_not_same_len;
unsigned long fallback_mod16;
unsigned long fallback_align16;
#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG
struct dentry *dbgfs_dir;
struct dentry *dbgfs_stats;
#endif
void *pctrl;
dma_addr_t dctrl;
};
struct sginfo {
u32 addr;
u32 len;
};
/*
* struct sl3516_ce_cipher_req_ctx - context for a skcipher request
* @t_src: list of mapped SGs with their size
* @t_dst: list of mapped SGs with their size
* @op_dir: direction (encrypt vs decrypt) for this request
* @pctrllen: the length of the ctrl packet
* @tqflag: the TQflag to set in data packet
* @h pointer to the pkt_control_cipher header
* @nr_sgs: number of source SG
* @nr_sgd: number of destination SG
* @fallback_req: request struct for invoking the fallback skcipher TFM
*/
struct sl3516_ce_cipher_req_ctx {
struct sginfo t_src[MAXDESC];
struct sginfo t_dst[MAXDESC];
u32 op_dir;
unsigned int pctrllen;
u32 tqflag;
struct pkt_control_cipher *h;
int nr_sgs;
int nr_sgd;
struct skcipher_request fallback_req; // keep at the end
};
/*
* struct sl3516_ce_cipher_tfm_ctx - context for a skcipher TFM
* @enginectx: crypto_engine used by this TFM
* @key: pointer to key data
* @keylen: len of the key
* @ce: pointer to the private data of driver handling this TFM
* @fallback_tfm: pointer to the fallback TFM
*
* enginectx must be the first element
*/
struct sl3516_ce_cipher_tfm_ctx {
struct crypto_engine_ctx enginectx;
u32 *key;
u32 keylen;
struct sl3516_ce_dev *ce;
struct crypto_skcipher *fallback_tfm;
};
/*
* struct sl3516_ce_alg_template - crypto_alg template
* @type: the CRYPTO_ALG_TYPE for this template
* @mode: value to be used in control packet for this algorithm
* @ce: pointer to the sl3516_ce_dev structure associated with
* this template
* @alg: one of sub struct must be used
* @stat_req: number of request done on this template
* @stat_fb: number of request which has fallbacked
* @stat_bytes: total data size done by this template
*/
struct sl3516_ce_alg_template {
u32 type;
u32 mode;
struct sl3516_ce_dev *ce;
union {
struct skcipher_alg skcipher;
} alg;
unsigned long stat_req;
unsigned long stat_fb;
unsigned long stat_bytes;
};
int sl3516_ce_enqueue(struct crypto_async_request *areq, u32 type);
int sl3516_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int keylen);
int sl3516_ce_cipher_init(struct crypto_tfm *tfm);
void sl3516_ce_cipher_exit(struct crypto_tfm *tfm);
int sl3516_ce_skdecrypt(struct skcipher_request *areq);
int sl3516_ce_skencrypt(struct skcipher_request *areq);
int sl3516_ce_run_task(struct sl3516_ce_dev *ce,
struct sl3516_ce_cipher_req_ctx *rctx, const char *name);
int sl3516_ce_rng_register(struct sl3516_ce_dev *ce);
void sl3516_ce_rng_unregister(struct sl3516_ce_dev *ce);

185
drivers/crypto/hisilicon/hpre/hpre_crypto.c
View File

@ -5,6 +5,7 @@
#include <crypto/dh.h>
#include <crypto/ecc_curve.h>
#include <crypto/ecdh.h>
#include <crypto/rng.h>
#include <crypto/internal/akcipher.h>
#include <crypto/internal/kpp.h>
#include <crypto/internal/rsa.h>
@ -30,7 +31,6 @@ struct hpre_ctx;
#define HPRE_DH_G_FLAG 0x02
#define HPRE_TRY_SEND_TIMES 100
#define HPRE_INVLD_REQ_ID (-1)
#define HPRE_DEV(ctx) (&((ctx)->qp->qm->pdev->dev))
#define HPRE_SQE_ALG_BITS 5
#define HPRE_SQE_DONE_SHIFT 30
@ -39,12 +39,17 @@ struct hpre_ctx;
#define HPRE_DFX_SEC_TO_US 1000000
#define HPRE_DFX_US_TO_NS 1000
/* due to nist p521 */
#define HPRE_ECC_MAX_KSZ 66
/* size in bytes of the n prime */
#define HPRE_ECC_NIST_P192_N_SIZE 24
#define HPRE_ECC_NIST_P256_N_SIZE 32
#define HPRE_ECC_NIST_P384_N_SIZE 48
/* size in bytes */
#define HPRE_ECC_HW256_KSZ_B 32
#define HPRE_ECC_HW384_KSZ_B 48
typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
@ -102,6 +107,7 @@ struct hpre_curve25519_ctx {
struct hpre_ctx {
struct hisi_qp *qp;
struct device *dev;
struct hpre_asym_request **req_list;
struct hpre *hpre;
spinlock_t req_lock;
@ -214,8 +220,7 @@ static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,
struct scatterlist *data, unsigned int len,
int is_src, dma_addr_t *tmp)
{
struct hpre_ctx *ctx = hpre_req->ctx;
struct device *dev = HPRE_DEV(ctx);
struct device *dev = hpre_req->ctx->dev;
enum dma_data_direction dma_dir;
if (is_src) {
@ -239,7 +244,7 @@ static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
int is_src, dma_addr_t *tmp)
{
struct hpre_ctx *ctx = hpre_req->ctx;
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
void *ptr;
int shift;
@ -293,11 +298,13 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
struct scatterlist *dst,
struct scatterlist *src)
{
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
struct hpre_sqe *sqe = &req->req;
dma_addr_t tmp;
tmp = le64_to_cpu(sqe->in);
if (unlikely(dma_mapping_error(dev, tmp)))
return;
if (src) {
if (req->src)
@ -307,6 +314,8 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
}
tmp = le64_to_cpu(sqe->out);
if (unlikely(dma_mapping_error(dev, tmp)))
return;
if (req->dst) {
if (dst)
@ -321,16 +330,15 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
void **kreq)
{
struct device *dev = HPRE_DEV(ctx);
struct hpre_asym_request *req;
unsigned int err, done, alg;
int id;
#define HPRE_NO_HW_ERR 0
#define HPRE_HW_TASK_DONE 3
#define HREE_HW_ERR_MASK 0x7ff
#define HREE_SQE_DONE_MASK 0x3
#define HREE_ALG_TYPE_MASK 0x1f
#define HREE_HW_ERR_MASK GENMASK(10, 0)
#define HREE_SQE_DONE_MASK GENMASK(1, 0)
#define HREE_ALG_TYPE_MASK GENMASK(4, 0)
id = (int)le16_to_cpu(sqe->tag);
req = ctx->req_list[id];
hpre_rm_req_from_ctx(req);
@ -346,7 +354,7 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
return 0;
alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;
dev_err_ratelimited(dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
alg, done, err);
return -EINVAL;
@ -361,6 +369,7 @@ static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
spin_lock_init(&ctx->req_lock);
ctx->qp = qp;
ctx->dev = &qp->qm->pdev->dev;
hpre = container_of(ctx->qp->qm, struct hpre, qm);
ctx->hpre = hpre;
@ -524,6 +533,8 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
}
msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
h_req->ctx = ctx;
@ -618,14 +629,14 @@ static int hpre_is_dh_params_length_valid(unsigned int key_sz)
case _HPRE_DH_GRP15:
case _HPRE_DH_GRP16:
return 0;
default:
return -EINVAL;
}
return -EINVAL;
}
static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
{
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
unsigned int sz;
if (params->p_size > HPRE_DH_MAX_P_SZ)
@ -664,7 +675,7 @@ static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
{
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
unsigned int sz = ctx->key_sz;
if (is_clear_all)
@ -877,18 +888,18 @@ static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
if (!hpre_rsa_key_size_is_support(ctx->key_sz))
return 0;
ctx->rsa.pubkey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1,
ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1,
&ctx->rsa.dma_pubkey,
GFP_KERNEL);
if (!ctx->rsa.pubkey)
return -ENOMEM;
if (private) {
ctx->rsa.prikey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1,
ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1,
&ctx->rsa.dma_prikey,
GFP_KERNEL);
if (!ctx->rsa.prikey) {
dma_free_coherent(HPRE_DEV(ctx), vlen << 1,
dma_free_coherent(ctx->dev, vlen << 1,
ctx->rsa.pubkey,
ctx->rsa.dma_pubkey);
ctx->rsa.pubkey = NULL;
@ -950,7 +961,7 @@ static int hpre_crt_para_get(char *para, size_t para_sz,
static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
{
unsigned int hlf_ksz = ctx->key_sz >> 1;
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
u64 offset;
int ret;
@ -1008,7 +1019,7 @@ static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
{
unsigned int half_key_sz = ctx->key_sz >> 1;
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
if (is_clear_all)
hisi_qm_stop_qp(ctx->qp);
@ -1179,7 +1190,7 @@ static void hpre_key_to_big_end(u8 *data, int len)
static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,
bool is_ecdh)
{
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
unsigned int sz = ctx->key_sz;
unsigned int shift = sz << 1;
@ -1202,12 +1213,21 @@ static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,
hpre_ctx_clear(ctx, is_clear_all);
}
/*
* The bits of 192/224/256/384/521 are supported by HPRE,
* and convert the bits like:
* bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576;
* If the parameter bit width is insufficient, then we fill in the
* high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8;
*/
static unsigned int hpre_ecdh_supported_curve(unsigned short id)
{
switch (id) {
case ECC_CURVE_NIST_P192:
case ECC_CURVE_NIST_P256:
return HPRE_ECC_HW256_KSZ_B;
case ECC_CURVE_NIST_P384:
return HPRE_ECC_HW384_KSZ_B;
default:
break;
}
@ -1272,6 +1292,8 @@ static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
return HPRE_ECC_NIST_P192_N_SIZE;
case ECC_CURVE_NIST_P256:
return HPRE_ECC_NIST_P256_N_SIZE;
case ECC_CURVE_NIST_P384:
return HPRE_ECC_NIST_P384_N_SIZE;
default:
break;
}
@ -1281,7 +1303,7 @@ static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)
{
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
unsigned int sz, shift, curve_sz;
int ret;
@ -1328,11 +1350,32 @@ static bool hpre_key_is_zero(char *key, unsigned short key_sz)
return true;
}
static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params)
{
struct device *dev = ctx->dev;
int ret;
ret = crypto_get_default_rng();
if (ret) {
dev_err(dev, "failed to get default rng, ret = %d!\n", ret);
return ret;
}
ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key,
params->key_size);
crypto_put_default_rng();
if (ret)
dev_err(dev, "failed to get rng, ret = %d!\n", ret);
return ret;
}
static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
unsigned int len)
{
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
char key[HPRE_ECC_MAX_KSZ];
unsigned int sz, sz_shift;
struct ecdh params;
int ret;
@ -1342,6 +1385,15 @@ static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
return -EINVAL;
}
/* Use stdrng to generate private key */
if (!params.key || !params.key_size) {
params.key = key;
params.key_size = hpre_ecdh_get_curvesz(ctx->curve_id);
ret = ecdh_gen_privkey(ctx, &params);
if (ret)
return ret;
}
if (hpre_key_is_zero(params.key, params.key_size)) {
dev_err(dev, "Invalid hpre key!\n");
return -EINVAL;
@ -1367,16 +1419,20 @@ static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
struct scatterlist *dst,
struct scatterlist *src)
{
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
struct hpre_sqe *sqe = &req->req;
dma_addr_t dma;
dma = le64_to_cpu(sqe->in);
if (unlikely(dma_mapping_error(dev, dma)))
return;
if (src && req->src)
dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
dma = le64_to_cpu(sqe->out);
if (unlikely(dma_mapping_error(dev, dma)))
return;
if (req->dst)
dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
@ -1431,6 +1487,8 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
h_req->areq.ecdh = req;
msg = &h_req->req;
memset(msg, 0, sizeof(*msg));
msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
msg->key = cpu_to_le64(ctx->ecdh.dma_p);
msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
@ -1450,7 +1508,7 @@ static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
{
struct hpre_sqe *msg = &hpre_req->req;
struct hpre_ctx *ctx = hpre_req->ctx;
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
unsigned int tmpshift;
dma_addr_t dma = 0;
void *ptr;
@ -1480,8 +1538,8 @@ static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
{
struct hpre_sqe *msg = &hpre_req->req;
struct hpre_ctx *ctx = hpre_req->ctx;
struct device *dev = HPRE_DEV(ctx);
dma_addr_t dma = 0;
struct device *dev = ctx->dev;
dma_addr_t dma;
if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
dev_err(dev, "data or data length is illegal!\n");
@ -1503,7 +1561,7 @@ static int hpre_ecdh_compute_value(struct kpp_request *req)
{
struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
void *tmp = kpp_request_ctx(req);
struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
struct hpre_sqe *msg = &hpre_req->req;
@ -1568,6 +1626,15 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
}
static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
{
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
ctx->curve_id = ECC_CURVE_NIST_P384;
return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
}
static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
{
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
@ -1609,7 +1676,7 @@ static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf,
static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf,
unsigned int len)
{
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
unsigned int sz = ctx->key_sz;
unsigned int shift = sz << 1;
@ -1634,7 +1701,7 @@ static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
unsigned int len)
{
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
int ret = -EINVAL;
if (len != CURVE25519_KEY_SIZE ||
@ -1662,16 +1729,20 @@ static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx,
struct scatterlist *dst,
struct scatterlist *src)
{
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
struct hpre_sqe *sqe = &req->req;
dma_addr_t dma;
dma = le64_to_cpu(sqe->in);
if (unlikely(dma_mapping_error(dev, dma)))
return;
if (src && req->src)
dma_free_coherent(dev, ctx->key_sz, req->src, dma);
dma = le64_to_cpu(sqe->out);
if (unlikely(dma_mapping_error(dev, dma)))
return;
if (req->dst)
dma_free_coherent(dev, ctx->key_sz, req->dst, dma);
@ -1722,6 +1793,8 @@ static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,
h_req->areq.curve25519 = req;
msg = &h_req->req;
memset(msg, 0, sizeof(*msg));
msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
msg->key = cpu_to_le64(ctx->curve25519.dma_p);
msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
@ -1752,7 +1825,7 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
{
struct hpre_sqe *msg = &hpre_req->req;
struct hpre_ctx *ctx = hpre_req->ctx;
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
u8 p[CURVE25519_KEY_SIZE] = { 0 };
const struct ecc_curve *curve;
dma_addr_t dma = 0;
@ -1790,8 +1863,12 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
* When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p,
* we get its modulus to p, and then use it.
*/
if (memcmp(ptr, p, ctx->key_sz) >= 0)
if (memcmp(ptr, p, ctx->key_sz) == 0) {
dev_err(dev, "gx is p!\n");
return -EINVAL;
} else if (memcmp(ptr, p, ctx->key_sz) > 0) {
hpre_curve25519_src_modulo_p(ptr);
}
hpre_req->src = ptr;
msg->in = cpu_to_le64(dma);
@ -1807,8 +1884,8 @@ static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req,
{
struct hpre_sqe *msg = &hpre_req->req;
struct hpre_ctx *ctx = hpre_req->ctx;
struct device *dev = HPRE_DEV(ctx);
dma_addr_t dma = 0;
struct device *dev = ctx->dev;
dma_addr_t dma;
if (!data || !sg_is_last(data) || len != ctx->key_sz) {
dev_err(dev, "data or data length is illegal!\n");
@ -1830,7 +1907,7 @@ static int hpre_curve25519_compute_value(struct kpp_request *req)
{
struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
struct device *dev = HPRE_DEV(ctx);
struct device *dev = ctx->dev;
void *tmp = kpp_request_ctx(req);
struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
struct hpre_sqe *msg = &hpre_req->req;
@ -1940,7 +2017,7 @@ static struct kpp_alg ecdh_nist_p192 = {
.cra_ctxsize = sizeof(struct hpre_ctx),
.cra_priority = HPRE_CRYPTO_ALG_PRI,
.cra_name = "ecdh-nist-p192",
.cra_driver_name = "hpre-ecdh",
.cra_driver_name = "hpre-ecdh-nist-p192",
.cra_module = THIS_MODULE,
},
};
@ -1957,7 +2034,24 @@ static struct kpp_alg ecdh_nist_p256 = {
.cra_ctxsize = sizeof(struct hpre_ctx),
.cra_priority = HPRE_CRYPTO_ALG_PRI,
.cra_name = "ecdh-nist-p256",
.cra_driver_name = "hpre-ecdh",
.cra_driver_name = "hpre-ecdh-nist-p256",
.cra_module = THIS_MODULE,
},
};
static struct kpp_alg ecdh_nist_p384 = {
.set_secret = hpre_ecdh_set_secret,
.generate_public_key = hpre_ecdh_compute_value,
.compute_shared_secret = hpre_ecdh_compute_value,
.max_size = hpre_ecdh_max_size,
.init = hpre_ecdh_nist_p384_init_tfm,
.exit = hpre_ecdh_exit_tfm,
.reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
.base = {
.cra_ctxsize = sizeof(struct hpre_ctx),
.cra_priority = HPRE_CRYPTO_ALG_PRI,
.cra_name = "ecdh-nist-p384",
.cra_driver_name = "hpre-ecdh-nist-p384",
.cra_module = THIS_MODULE,
},
};
@ -1989,16 +2083,25 @@ static int hpre_register_ecdh(void)
return ret;
ret = crypto_register_kpp(&ecdh_nist_p256);
if (ret) {
crypto_unregister_kpp(&ecdh_nist_p192);
return ret;
}
if (ret)
goto unregister_ecdh_p192;
ret = crypto_register_kpp(&ecdh_nist_p384);
if (ret)
goto unregister_ecdh_p256;
return 0;
unregister_ecdh_p256:
crypto_unregister_kpp(&ecdh_nist_p256);
unregister_ecdh_p192:
crypto_unregister_kpp(&ecdh_nist_p192);
return ret;
}
static void hpre_unregister_ecdh(void)
{
crypto_unregister_kpp(&ecdh_nist_p384);
crypto_unregister_kpp(&ecdh_nist_p256);
crypto_unregister_kpp(&ecdh_nist_p192);
}

252
drivers/crypto/hisilicon/hpre/hpre_main.c
View File

@ -36,7 +36,7 @@
#define HPRE_INT_MASK 0x301400
#define HPRE_INT_STATUS 0x301800
#define HPRE_CORE_INT_ENABLE 0
#define HPRE_CORE_INT_DISABLE 0x003fffff
#define HPRE_CORE_INT_DISABLE GENMASK(21, 0)
#define HPRE_RDCHN_INI_ST 0x301a00
#define HPRE_CLSTR_BASE 0x302000
#define HPRE_CORE_EN_OFFSET 0x04
@ -50,6 +50,7 @@
#define HPRE_RAS_NFE_ENB 0x301414
#define HPRE_HAC_RAS_NFE_ENABLE 0x3ffffe
#define HPRE_RAS_FE_ENB 0x301418
#define HPRE_OOO_SHUTDOWN_SEL 0x301a3c
#define HPRE_HAC_RAS_FE_ENABLE 0
#define HPRE_CORE_ENB (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET)
@ -57,7 +58,6 @@
#define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET)
#define HPRE_HAC_ECC1_CNT 0x301a04
#define HPRE_HAC_ECC2_CNT 0x301a08
#define HPRE_HAC_INT_STATUS 0x301800
#define HPRE_HAC_SOURCE_INT 0x301600
#define HPRE_CLSTR_ADDR_INTRVL 0x1000
#define HPRE_CLUSTER_INQURY 0x100
@ -69,13 +69,17 @@
#define HPRE_DBGFS_VAL_MAX_LEN 20
#define HPRE_PCI_DEVICE_ID 0xa258
#define HPRE_PCI_VF_DEVICE_ID 0xa259
#define HPRE_ADDR(qm, offset) ((qm)->io_base + (offset))
#define HPRE_QM_USR_CFG_MASK 0xfffffffe
#define HPRE_QM_AXI_CFG_MASK 0xffff
#define HPRE_QM_VFG_AX_MASK 0xff
#define HPRE_BD_USR_MASK 0x3
#define HPRE_CLUSTER_CORE_MASK_V2 0xf
#define HPRE_CLUSTER_CORE_MASK_V3 0xff
#define HPRE_QM_USR_CFG_MASK GENMASK(31, 1)
#define HPRE_QM_AXI_CFG_MASK GENMASK(15, 0)
#define HPRE_QM_VFG_AX_MASK GENMASK(7, 0)
#define HPRE_BD_USR_MASK GENMASK(1, 0)
#define HPRE_CLUSTER_CORE_MASK_V2 GENMASK(3, 0)
#define HPRE_CLUSTER_CORE_MASK_V3 GENMASK(7, 0)
#define HPRE_PREFETCH_CFG 0x301130
#define HPRE_SVA_PREFTCH_DFX 0x30115C
#define HPRE_PREFETCH_ENABLE (~(BIT(0) | BIT(30)))
#define HPRE_PREFETCH_DISABLE BIT(30)
#define HPRE_SVA_DISABLE_READY (BIT(4) | BIT(8))
#define HPRE_AM_OOO_SHUTDOWN_ENB 0x301044
#define HPRE_AM_OOO_SHUTDOWN_ENABLE BIT(0)
@ -88,11 +92,7 @@
#define HPRE_QM_PM_FLR BIT(11)
#define HPRE_QM_SRIOV_FLR BIT(12)
#define HPRE_CLUSTERS_NUM(qm) \
(((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : HPRE_CLUSTERS_NUM_V2)
#define HPRE_CLUSTER_CORE_MASK(qm) \
(((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTER_CORE_MASK_V3 :\
HPRE_CLUSTER_CORE_MASK_V2)
#define HPRE_SHAPER_TYPE_RATE 128
#define HPRE_VIA_MSI_DSM 1
#define HPRE_SQE_MASK_OFFSET 8
#define HPRE_SQE_MASK_LEN 24
@ -123,21 +123,49 @@ static const char * const hpre_debug_file_name[] = {
};
static const struct hpre_hw_error hpre_hw_errors[] = {
{ .int_msk = BIT(0), .msg = "core_ecc_1bit_err_int_set" },
{ .int_msk = BIT(1), .msg = "core_ecc_2bit_err_int_set" },
{ .int_msk = BIT(2), .msg = "dat_wb_poison_int_set" },
{ .int_msk = BIT(3), .msg = "dat_rd_poison_int_set" },
{ .int_msk = BIT(4), .msg = "bd_rd_poison_int_set" },
{ .int_msk = BIT(5), .msg = "ooo_ecc_2bit_err_int_set" },
{ .int_msk = BIT(6), .msg = "cluster1_shb_timeout_int_set" },
{ .int_msk = BIT(7), .msg = "cluster2_shb_timeout_int_set" },
{ .int_msk = BIT(8), .msg = "cluster3_shb_timeout_int_set" },
{ .int_msk = BIT(9), .msg = "cluster4_shb_timeout_int_set" },
{ .int_msk = GENMASK(15, 10), .msg = "ooo_rdrsp_err_int_set" },
{ .int_msk = GENMASK(21, 16), .msg = "ooo_wrrsp_err_int_set" },
{ .int_msk = BIT(22), .msg = "pt_rng_timeout_int_set"},
{ .int_msk = BIT(23), .msg = "sva_fsm_timeout_int_set"},
{
.int_msk = BIT(0),
.msg = "core_ecc_1bit_err_int_set"
}, {
.int_msk = BIT(1),
.msg = "core_ecc_2bit_err_int_set"
}, {
.int_msk = BIT(2),
.msg = "dat_wb_poison_int_set"
}, {
.int_msk = BIT(3),
.msg = "dat_rd_poison_int_set"
}, {
.int_msk = BIT(4),
.msg = "bd_rd_poison_int_set"
}, {
.int_msk = BIT(5),
.msg = "ooo_ecc_2bit_err_int_set"
}, {
.int_msk = BIT(6),
.msg = "cluster1_shb_timeout_int_set"
}, {
.int_msk = BIT(7),
.msg = "cluster2_shb_timeout_int_set"
}, {
.int_msk = BIT(8),
.msg = "cluster3_shb_timeout_int_set"
}, {
.int_msk = BIT(9),
.msg = "cluster4_shb_timeout_int_set"
}, {
.int_msk = GENMASK(15, 10),
.msg = "ooo_rdrsp_err_int_set"
}, {
.int_msk = GENMASK(21, 16),
.msg = "ooo_wrrsp_err_int_set"
}, {
.int_msk = BIT(22),
.msg = "pt_rng_timeout_int_set"
}, {
.int_msk = BIT(23),
.msg = "sva_fsm_timeout_int_set"
}, {
/* sentinel */
}
};
@ -224,6 +252,18 @@ static u32 vfs_num;
module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
static inline int hpre_cluster_num(struct hisi_qm *qm)
{
return (qm->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 :
HPRE_CLUSTERS_NUM_V2;
}
static inline int hpre_cluster_core_mask(struct hisi_qm *qm)
{
return (qm->ver >= QM_HW_V3) ?
HPRE_CLUSTER_CORE_MASK_V3 : HPRE_CLUSTER_CORE_MASK_V2;
}
struct hisi_qp *hpre_create_qp(u8 type)
{
int node = cpu_to_node(smp_processor_id());
@ -290,8 +330,8 @@ static int hpre_cfg_by_dsm(struct hisi_qm *qm)
static int hpre_set_cluster(struct hisi_qm *qm)
{
u32 cluster_core_mask = HPRE_CLUSTER_CORE_MASK(qm);
u8 clusters_num = HPRE_CLUSTERS_NUM(qm);
u32 cluster_core_mask = hpre_cluster_core_mask(qm);
u8 clusters_num = hpre_cluster_num(qm);
struct device *dev = &qm->pdev->dev;
unsigned long offset;
u32 val = 0;
@ -302,10 +342,10 @@ static int hpre_set_cluster(struct hisi_qm *qm)
/* clusters initiating */
writel(cluster_core_mask,
HPRE_ADDR(qm, offset + HPRE_CORE_ENB));
writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG));
ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset +
HPRE_CORE_INI_STATUS), val,
qm->io_base + offset + HPRE_CORE_ENB);
writel(0x1, qm->io_base + offset + HPRE_CORE_INI_CFG);
ret = readl_relaxed_poll_timeout(qm->io_base + offset +
HPRE_CORE_INI_STATUS, val,
((val & cluster_core_mask) ==
cluster_core_mask),
HPRE_REG_RD_INTVRL_US,
@ -329,11 +369,52 @@ static void disable_flr_of_bme(struct hisi_qm *qm)
{
u32 val;
val = readl(HPRE_ADDR(qm, QM_PEH_AXUSER_CFG));
val = readl(qm->io_base + QM_PEH_AXUSER_CFG);
val &= ~(HPRE_QM_BME_FLR | HPRE_QM_SRIOV_FLR);
val |= HPRE_QM_PM_FLR;
writel(val, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG));
writel(PEH_AXUSER_CFG_ENABLE, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG_ENABLE));
writel(val, qm->io_base + QM_PEH_AXUSER_CFG);
writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE);
}
static void hpre_open_sva_prefetch(struct hisi_qm *qm)
{
u32 val;
int ret;
if (qm->ver < QM_HW_V3)
return;
/* Enable prefetch */
val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG);
val &= HPRE_PREFETCH_ENABLE;
writel(val, qm->io_base + HPRE_PREFETCH_CFG);
ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG,
val, !(val & HPRE_PREFETCH_DISABLE),
HPRE_REG_RD_INTVRL_US,
HPRE_REG_RD_TMOUT_US);
if (ret)
pci_err(qm->pdev, "failed to open sva prefetch\n");
}
static void hpre_close_sva_prefetch(struct hisi_qm *qm)
{
u32 val;
int ret;
if (qm->ver < QM_HW_V3)
return;
val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG);
val |= HPRE_PREFETCH_DISABLE;
writel(val, qm->io_base + HPRE_PREFETCH_CFG);
ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX,
val, !(val & HPRE_SVA_DISABLE_READY),
HPRE_REG_RD_INTVRL_US,
HPRE_REG_RD_TMOUT_US);
if (ret)
pci_err(qm->pdev, "failed to close sva prefetch\n");
}
static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
@ -342,33 +423,33 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
u32 val;
int ret;
writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_ARUSER_M_CFG_ENABLE));
writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_AWUSER_M_CFG_ENABLE));
writel_relaxed(HPRE_QM_AXI_CFG_MASK, HPRE_ADDR(qm, QM_AXI_M_CFG));
writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_ARUSER_M_CFG_ENABLE);
writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_AWUSER_M_CFG_ENABLE);
writel_relaxed(HPRE_QM_AXI_CFG_MASK, qm->io_base + QM_AXI_M_CFG);
/* HPRE need more time, we close this interrupt */
val = readl_relaxed(HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK));
val = readl_relaxed(qm->io_base + HPRE_QM_ABNML_INT_MASK);
val |= BIT(HPRE_TIMEOUT_ABNML_BIT);
writel_relaxed(val, HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK));
writel_relaxed(val, qm->io_base + HPRE_QM_ABNML_INT_MASK);
if (qm->ver >= QM_HW_V3)
writel(HPRE_RSA_ENB | HPRE_ECC_ENB,
HPRE_ADDR(qm, HPRE_TYPES_ENB));
qm->io_base + HPRE_TYPES_ENB);
else
writel(HPRE_RSA_ENB, HPRE_ADDR(qm, HPRE_TYPES_ENB));
writel(HPRE_RSA_ENB, qm->io_base + HPRE_TYPES_ENB);
writel(HPRE_QM_VFG_AX_MASK, HPRE_ADDR(qm, HPRE_VFG_AXCACHE));
writel(0x0, HPRE_ADDR(qm, HPRE_BD_ENDIAN));
writel(0x0, HPRE_ADDR(qm, HPRE_INT_MASK));
writel(0x0, HPRE_ADDR(qm, HPRE_POISON_BYPASS));
writel(0x0, HPRE_ADDR(qm, HPRE_COMM_CNT_CLR_CE));
writel(0x0, HPRE_ADDR(qm, HPRE_ECC_BYPASS));
writel(HPRE_QM_VFG_AX_MASK, qm->io_base + HPRE_VFG_AXCACHE);
writel(0x0, qm->io_base + HPRE_BD_ENDIAN);
writel(0x0, qm->io_base + HPRE_INT_MASK);
writel(0x0, qm->io_base + HPRE_POISON_BYPASS);
writel(0x0, qm->io_base + HPRE_COMM_CNT_CLR_CE);
writel(0x0, qm->io_base + HPRE_ECC_BYPASS);
writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_ARUSR_CFG));
writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_AWUSR_CFG));
writel(0x1, HPRE_ADDR(qm, HPRE_RDCHN_INI_CFG));
ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, HPRE_RDCHN_INI_ST), val,
val & BIT(0),
writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_ARUSR_CFG);
writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_AWUSR_CFG);
writel(0x1, qm->io_base + HPRE_RDCHN_INI_CFG);
ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_RDCHN_INI_ST, val,
val & BIT(0),
HPRE_REG_RD_INTVRL_US,
HPRE_REG_RD_TMOUT_US);
if (ret) {
@ -397,7 +478,7 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
static void hpre_cnt_regs_clear(struct hisi_qm *qm)
{
u8 clusters_num = HPRE_CLUSTERS_NUM(qm);
u8 clusters_num = hpre_cluster_num(qm);
unsigned long offset;
int i;
@ -413,36 +494,49 @@ static void hpre_cnt_regs_clear(struct hisi_qm *qm)
hisi_qm_debug_regs_clear(qm);
}
static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable)
{
u32 val1, val2;
val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
if (enable) {
val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE;
val2 = HPRE_HAC_RAS_NFE_ENABLE;
} else {
val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE;
val2 = 0x0;
}
if (qm->ver > QM_HW_V2)
writel(val2, qm->io_base + HPRE_OOO_SHUTDOWN_SEL);
writel(val1, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
}
static void hpre_hw_error_disable(struct hisi_qm *qm)
{
u32 val;
/* disable hpre hw error interrupts */
writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK);
/* disable HPRE block master OOO when m-bit error occur */
val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
val &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE;
writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
/* disable HPRE block master OOO when nfe occurs on Kunpeng930 */
hpre_master_ooo_ctrl(qm, false);
}
static void hpre_hw_error_enable(struct hisi_qm *qm)
{
u32 val;
/* clear HPRE hw error source if having */
writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT);
/* enable hpre hw error interrupts */
writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK);
/* configure error type */
writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB);
writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB);
writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB);
/* enable HPRE block master OOO when m-bit error occur */
val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
val |= HPRE_AM_OOO_SHUTDOWN_ENABLE;
writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
/* enable HPRE block master OOO when nfe occurs on Kunpeng930 */
hpre_master_ooo_ctrl(qm, true);
/* enable hpre hw error interrupts */
writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK);
}
static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file)
@ -650,7 +744,7 @@ static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm)
static int hpre_cluster_debugfs_init(struct hisi_qm *qm)
{
u8 clusters_num = HPRE_CLUSTERS_NUM(qm);
u8 clusters_num = hpre_cluster_num(qm);
struct device *dev = &qm->pdev->dev;
char buf[HPRE_DBGFS_VAL_MAX_LEN];
struct debugfs_regset32 *regset;
@ -788,7 +882,7 @@ static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts)
static u32 hpre_get_hw_err_status(struct hisi_qm *qm)
{
return readl(qm->io_base + HPRE_HAC_INT_STATUS);
return readl(qm->io_base + HPRE_INT_STATUS);
}
static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
@ -802,9 +896,9 @@ static void hpre_open_axi_master_ooo(struct hisi_qm *qm)
value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE,
HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB));
qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE,
HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB));
qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
}
static void hpre_err_info_init(struct hisi_qm *qm)
@ -829,6 +923,8 @@ static const struct hisi_qm_err_ini hpre_err_ini = {
.clear_dev_hw_err_status = hpre_clear_hw_err_status,
.log_dev_hw_err = hpre_log_hw_error,
.open_axi_master_ooo = hpre_open_axi_master_ooo,
.open_sva_prefetch = hpre_open_sva_prefetch,
.close_sva_prefetch = hpre_close_sva_prefetch,
.err_info_init = hpre_err_info_init,
};
@ -841,6 +937,8 @@ static int hpre_pf_probe_init(struct hpre *hpre)
if (ret)
return ret;
hpre_open_sva_prefetch(qm);
qm->err_ini = &hpre_err_ini;
qm->err_ini->err_info_init(qm);
hisi_qm_dev_err_init(qm);
@ -850,6 +948,7 @@ static int hpre_pf_probe_init(struct hpre *hpre)
static int hpre_probe_init(struct hpre *hpre)
{
u32 type_rate = HPRE_SHAPER_TYPE_RATE;
struct hisi_qm *qm = &hpre->qm;
int ret;
@ -857,6 +956,11 @@ static int hpre_probe_init(struct hpre *hpre)
ret = hpre_pf_probe_init(hpre);
if (ret)
return ret;
/* Enable shaper type 0 */
if (qm->ver >= QM_HW_V3) {
type_rate |= QM_SHAPER_ENABLE;
qm->type_rate = type_rate;
}
}
return 0;

1843
drivers/crypto/hisilicon/qm.c
View File

File diff suppressed because it is too large Load Diff

17
drivers/crypto/hisilicon/qm.h
View File

@ -76,6 +76,9 @@
#define QM_Q_DEPTH 1024
#define QM_MIN_QNUM 2
#define HISI_ACC_SGL_SGE_NR_MAX 255
#define QM_SHAPER_CFG 0x100164
#define QM_SHAPER_ENABLE BIT(30)
#define QM_SHAPER_TYPE1_OFFSET 10
/* page number for queue file region */
#define QM_DOORBELL_PAGE_NR 1
@ -148,6 +151,14 @@ struct qm_debug {
struct debugfs_file files[DEBUG_FILE_NUM];
};
struct qm_shaper_factor {
u32 func_qos;
u64 cir_b;
u64 cir_u;
u64 cir_s;
u64 cbs_s;
};
struct qm_dma {
void *va;
dma_addr_t dma;
@ -188,6 +199,8 @@ struct hisi_qm_err_ini {
void (*clear_dev_hw_err_status)(struct hisi_qm *qm, u32 err_sts);
void (*open_axi_master_ooo)(struct hisi_qm *qm);
void (*close_axi_master_ooo)(struct hisi_qm *qm);
void (*open_sva_prefetch)(struct hisi_qm *qm);
void (*close_sva_prefetch)(struct hisi_qm *qm);
void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts);
void (*err_info_init)(struct hisi_qm *qm);
};
@ -248,6 +261,7 @@ struct hisi_qm {
struct workqueue_struct *wq;
struct work_struct work;
struct work_struct rst_work;
struct work_struct cmd_process;
const char *algs;
bool use_sva;
@ -259,6 +273,9 @@ struct hisi_qm {
resource_size_t db_phys_base;
struct uacce_device *uacce;
int mode;
struct qm_shaper_factor *factor;
u32 mb_qos;
u32 type_rate;
};
struct hisi_qp_status {

23
drivers/crypto/hisilicon/sec2/sec.h
View File

@ -13,14 +13,14 @@ struct sec_alg_res {
dma_addr_t pbuf_dma;
u8 *c_ivin;
dma_addr_t c_ivin_dma;
u8 *a_ivin;
dma_addr_t a_ivin_dma;
u8 *out_mac;
dma_addr_t out_mac_dma;
};
/* Cipher request of SEC private */
struct sec_cipher_req {
struct hisi_acc_hw_sgl *c_in;
dma_addr_t c_in_dma;
struct hisi_acc_hw_sgl *c_out;
dma_addr_t c_out_dma;
u8 *c_ivin;
@ -33,15 +33,25 @@ struct sec_cipher_req {
struct sec_aead_req {
u8 *out_mac;
dma_addr_t out_mac_dma;
u8 *a_ivin;
dma_addr_t a_ivin_dma;
struct aead_request *aead_req;
};
/* SEC request of Crypto */
struct sec_req {
struct sec_sqe sec_sqe;
union {
struct sec_sqe sec_sqe;
struct sec_sqe3 sec_sqe3;
};
struct sec_ctx *ctx;
struct sec_qp_ctx *qp_ctx;
/**
* Common parameter of the SEC request.
*/
struct hisi_acc_hw_sgl *in;
dma_addr_t in_dma;
struct sec_cipher_req c_req;
struct sec_aead_req aead_req;
struct list_head backlog_head;
@ -81,7 +91,9 @@ struct sec_auth_ctx {
u8 a_key_len;
u8 mac_len;
u8 a_alg;
bool fallback;
struct crypto_shash *hash_tfm;
struct crypto_aead *fallback_aead_tfm;
};
/* SEC cipher context which cipher's relatives */
@ -94,6 +106,10 @@ struct sec_cipher_ctx {
u8 c_mode;
u8 c_alg;
u8 c_key_len;
/* add software support */
bool fallback;
struct crypto_sync_skcipher *fbtfm;
};
/* SEC queue context which defines queue's relatives */
@ -137,6 +153,7 @@ struct sec_ctx {
bool pbuf_supported;
struct sec_cipher_ctx c_ctx;
struct sec_auth_ctx a_ctx;
u8 type_supported;
struct device *dev;
};

1044
drivers/crypto/hisilicon/sec2/sec_crypto.c
View File

File diff suppressed because it is too large Load Diff

193
drivers/crypto/hisilicon/sec2/sec_crypto.h
View File

@ -4,9 +4,11 @@
#ifndef __HISI_SEC_V2_CRYPTO_H
#define __HISI_SEC_V2_CRYPTO_H
#define SEC_AIV_SIZE 12
#define SEC_IV_SIZE 24
#define SEC_MAX_KEY_SIZE 64
#define SEC_COMM_SCENE 0
#define SEC_MIN_BLOCK_SZ 1
enum sec_calg {
SEC_CALG_3DES = 0x1,
@ -21,6 +23,11 @@ enum sec_hash_alg {
};
enum sec_mac_len {
SEC_HMAC_CCM_MAC = 16,
SEC_HMAC_GCM_MAC = 16,
SEC_SM3_MAC = 32,
SEC_HMAC_SM3_MAC = 32,
SEC_HMAC_MD5_MAC = 16,
SEC_HMAC_SHA1_MAC = 20,
SEC_HMAC_SHA256_MAC = 32,
SEC_HMAC_SHA512_MAC = 64,
@ -29,7 +36,11 @@ enum sec_mac_len {
enum sec_cmode {
SEC_CMODE_ECB = 0x0,
SEC_CMODE_CBC = 0x1,
SEC_CMODE_CFB = 0x2,
SEC_CMODE_OFB = 0x3,
SEC_CMODE_CTR = 0x4,
SEC_CMODE_CCM = 0x5,
SEC_CMODE_GCM = 0x6,
SEC_CMODE_XTS = 0x7,
};
@ -44,6 +55,7 @@ enum sec_ckey_type {
enum sec_bd_type {
SEC_BD_TYPE1 = 0x1,
SEC_BD_TYPE2 = 0x2,
SEC_BD_TYPE3 = 0x3,
};
enum sec_auth {
@ -63,6 +75,24 @@ enum sec_addr_type {
SEC_PRP = 0x2,
};
struct bd_status {
u64 tag;
u8 done;
u8 err_type;
u16 flag;
u16 icv;
};
enum {
AUTHPAD_PAD,
AUTHPAD_NOPAD,
};
enum {
AIGEN_GEN,
AIGEN_NOGEN,
};
struct sec_sqe_type2 {
/*
* mac_len: 0~4 bits
@ -209,6 +239,169 @@ struct sec_sqe {
struct sec_sqe_type2 type2;
};
struct bd3_auth_ivin {
__le64 a_ivin_addr;
__le32 rsvd0;
__le32 rsvd1;
} __packed __aligned(4);
struct bd3_skip_data {
__le32 rsvd0;
/*
* gran_num: 0~15 bits
* reserved: 16~31 bits
*/
__le32 gran_num;
/*
* src_skip_data_len: 0~24 bits
* reserved: 25~31 bits
*/
__le32 src_skip_data_len;
/*
* dst_skip_data_len: 0~24 bits
* reserved: 25~31 bits
*/
__le32 dst_skip_data_len;
};
struct bd3_stream_scene {
__le64 c_ivin_addr;
__le64 long_a_data_len;
/*
* auth_pad: 0~1 bits
* stream_protocol: 2~4 bits
* reserved: 5~7 bits
*/
__u8 stream_auth_pad;
__u8 plaintext_type;
__le16 pad_len_1p3;
} __packed __aligned(4);
struct bd3_no_scene {
__le64 c_ivin_addr;
__le32 rsvd0;
__le32 rsvd1;
__le32 rsvd2;
} __packed __aligned(4);
struct bd3_check_sum {
__u8 rsvd0;
__u8 hac_sva_status;
__le16 check_sum_i;
};
struct bd3_tls_type_back {
__u8 tls_1p3_type_back;
__u8 hac_sva_status;
__le16 pad_len_1p3_back;
};
struct sec_sqe3 {
/*
* type: 0~3 bit
* bd_invalid: 4 bit
* scene: 5~8 bit
* de: 9~10 bit
* src_addr_type: 11~13 bit
* dst_addr_type: 14~16 bit
* mac_addr_type: 17~19 bit
* reserved: 20~31 bits
*/
__le32 bd_param;
/*
* cipher: 0~1 bits
* ci_gen: 2~3 bit
* c_icv_len: 4~9 bit
* c_width: 10~12 bits
* c_key_len: 13~15 bits
*/
__le16 c_icv_key;
/*
* c_mode : 0~3 bits
* c_alg : 4~7 bits
*/
__u8 c_mode_alg;
/*
* nonce_len : 0~3 bits
* huk : 4 bits
* cal_iv_addr_en : 5 bits
* seq : 6 bits
* reserved : 7 bits
*/
__u8 huk_iv_seq;
__le64 tag;
__le64 data_src_addr;
__le64 a_key_addr;
union {
struct bd3_auth_ivin auth_ivin;
struct bd3_skip_data skip_data;
};
__le64 c_key_addr;
/*
* auth: 0~1 bits
* ai_gen: 2~3 bits
* mac_len: 4~8 bits
* akey_len: 9~14 bits
* a_alg: 15~20 bits
* key_sel: 21~24 bits
* updata_key: 25 bits
* reserved: 26~31 bits
*/
__le32 auth_mac_key;
__le32 salt;
__le16 auth_src_offset;
__le16 cipher_src_offset;
/*
* auth_len: 0~23 bit
* auth_key_offset: 24~31 bits
*/
__le32 a_len_key;
/*
* cipher_len: 0~23 bit
* auth_ivin_offset: 24~31 bits
*/
__le32 c_len_ivin;
__le64 data_dst_addr;
__le64 mac_addr;
union {
struct bd3_stream_scene stream_scene;
struct bd3_no_scene no_scene;
};
/*
* done: 0 bit
* icv: 1~3 bit
* csc: 4~6 bit
* flag: 7~10 bit
* reserved: 11~15 bit
*/
__le16 done_flag;
__u8 error_type;
__u8 warning_type;
union {
__le32 mac_i;
__le32 kek_key_addr_l;
};
union {
__le32 kek_key_addr_h;
struct bd3_check_sum check_sum;
struct bd3_tls_type_back tls_type_back;
};
__le32 counter;
} __packed __aligned(4);
int sec_register_to_crypto(struct hisi_qm *qm);
void sec_unregister_from_crypto(struct hisi_qm *qm);
#endif

100
drivers/crypto/hisilicon/sec2/sec_main.c
View File

@ -52,6 +52,7 @@
#define SEC_RAS_CE_ENB_MSK 0x88
#define SEC_RAS_FE_ENB_MSK 0x0
#define SEC_RAS_NFE_ENB_MSK 0x7c177
#define SEC_OOO_SHUTDOWN_SEL 0x301014
#define SEC_RAS_DISABLE 0x0
#define SEC_MEM_START_INIT_REG 0x301100
#define SEC_MEM_INIT_DONE_REG 0x301104
@ -84,6 +85,12 @@
#define SEC_USER1_SMMU_MASK (~SEC_USER1_SVA_SET)
#define SEC_CORE_INT_STATUS_M_ECC BIT(2)
#define SEC_PREFETCH_CFG 0x301130
#define SEC_SVA_TRANS 0x301EC4
#define SEC_PREFETCH_ENABLE (~(BIT(0) | BIT(1) | BIT(11)))
#define SEC_PREFETCH_DISABLE BIT(1)
#define SEC_SVA_DISABLE_READY (BIT(7) | BIT(11))
#define SEC_DELAY_10_US 10
#define SEC_POLL_TIMEOUT_US 1000
#define SEC_DBGFS_VAL_MAX_LEN 20
@ -91,6 +98,7 @@
#define SEC_SQE_MASK_OFFSET 64
#define SEC_SQE_MASK_LEN 48
#define SEC_SHAPER_TYPE_RATE 128
struct sec_hw_error {
u32 int_msk;
@ -331,6 +339,42 @@ static u8 sec_get_endian(struct hisi_qm *qm)
return SEC_64BE;
}
static void sec_open_sva_prefetch(struct hisi_qm *qm)
{
u32 val;
int ret;
if (qm->ver < QM_HW_V3)
return;
/* Enable prefetch */
val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG);
val &= SEC_PREFETCH_ENABLE;
writel(val, qm->io_base + SEC_PREFETCH_CFG);
ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG,
val, !(val & SEC_PREFETCH_DISABLE),
SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US);
if (ret)
pci_err(qm->pdev, "failed to open sva prefetch\n");
}
static void sec_close_sva_prefetch(struct hisi_qm *qm)
{
u32 val;
int ret;
val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG);
val |= SEC_PREFETCH_DISABLE;
writel(val, qm->io_base + SEC_PREFETCH_CFG);
ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS,
val, !(val & SEC_SVA_DISABLE_READY),
SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US);
if (ret)
pci_err(qm->pdev, "failed to close sva prefetch\n");
}
static int sec_engine_init(struct hisi_qm *qm)
{
int ret;
@ -430,53 +474,60 @@ static void sec_debug_regs_clear(struct hisi_qm *qm)
hisi_qm_debug_regs_clear(qm);
}
static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable)
{
u32 val1, val2;
val1 = readl(qm->io_base + SEC_CONTROL_REG);
if (enable) {
val1 |= SEC_AXI_SHUTDOWN_ENABLE;
val2 = SEC_RAS_NFE_ENB_MSK;
} else {
val1 &= SEC_AXI_SHUTDOWN_DISABLE;
val2 = 0x0;
}
if (qm->ver > QM_HW_V2)
writel(val2, qm->io_base + SEC_OOO_SHUTDOWN_SEL);
writel(val1, qm->io_base + SEC_CONTROL_REG);
}
static void sec_hw_error_enable(struct hisi_qm *qm)
{
u32 val;
if (qm->ver == QM_HW_V1) {
writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);
pci_info(qm->pdev, "V1 not support hw error handle\n");
return;
}
val = readl(qm->io_base + SEC_CONTROL_REG);
/* clear SEC hw error source if having */
writel(SEC_CORE_INT_CLEAR, qm->io_base + SEC_CORE_INT_SOURCE);
/* enable SEC hw error interrupts */
writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK);
/* enable RAS int */
writel(SEC_RAS_CE_ENB_MSK, qm->io_base + SEC_RAS_CE_REG);
writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG);
writel(SEC_RAS_NFE_ENB_MSK, qm->io_base + SEC_RAS_NFE_REG);
/* enable SEC block master OOO when m-bit error occur */
val = val | SEC_AXI_SHUTDOWN_ENABLE;
/* enable SEC block master OOO when nfe occurs on Kunpeng930 */
sec_master_ooo_ctrl(qm, true);
writel(val, qm->io_base + SEC_CONTROL_REG);
/* enable SEC hw error interrupts */
writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK);
}
static void sec_hw_error_disable(struct hisi_qm *qm)
{
u32 val;
/* disable SEC hw error interrupts */
writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);
val = readl(qm->io_base + SEC_CONTROL_REG);
/* disable SEC block master OOO when nfe occurs on Kunpeng930 */
sec_master_ooo_ctrl(qm, false);
/* disable RAS int */
writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG);
writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG);
writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG);
/* disable SEC hw error interrupts */
writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);
/* disable SEC block master OOO when m-bit error occur */
val = val & SEC_AXI_SHUTDOWN_DISABLE;
writel(val, qm->io_base + SEC_CONTROL_REG);
}
static u32 sec_clear_enable_read(struct sec_debug_file *file)
@ -743,6 +794,8 @@ static const struct hisi_qm_err_ini sec_err_ini = {
.clear_dev_hw_err_status = sec_clear_hw_err_status,
.log_dev_hw_err = sec_log_hw_error,
.open_axi_master_ooo = sec_open_axi_master_ooo,
.open_sva_prefetch = sec_open_sva_prefetch,
.close_sva_prefetch = sec_close_sva_prefetch,
.err_info_init = sec_err_info_init,
};
@ -758,6 +811,7 @@ static int sec_pf_probe_init(struct sec_dev *sec)
if (ret)
return ret;
sec_open_sva_prefetch(qm);
hisi_qm_dev_err_init(qm);
sec_debug_regs_clear(qm);
@ -821,6 +875,7 @@ static void sec_qm_uninit(struct hisi_qm *qm)
static int sec_probe_init(struct sec_dev *sec)
{
u32 type_rate = SEC_SHAPER_TYPE_RATE;
struct hisi_qm *qm = &sec->qm;
int ret;
@ -828,6 +883,11 @@ static int sec_probe_init(struct sec_dev *sec)
ret = sec_pf_probe_init(sec);
if (ret)
return ret;
/* enable shaper type 0 */
if (qm->ver >= QM_HW_V3) {
type_rate |= QM_SHAPER_ENABLE;
qm->type_rate = type_rate;
}
}
return 0;

99
drivers/crypto/hisilicon/zip/zip_main.c
View File

@ -68,6 +68,7 @@
#define HZIP_CORE_INT_RAS_CE_ENABLE 0x1
#define HZIP_CORE_INT_RAS_NFE_ENB 0x301164
#define HZIP_CORE_INT_RAS_FE_ENB 0x301168
#define HZIP_OOO_SHUTDOWN_SEL 0x30120C
#define HZIP_CORE_INT_RAS_NFE_ENABLE 0x1FFE
#define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16
#define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24
@ -96,6 +97,16 @@
#define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \
HZIP_RO_CNT_CLR_CE_EN)
#define HZIP_PREFETCH_CFG 0x3011B0
#define HZIP_SVA_TRANS 0x3011C4
#define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0)))
#define HZIP_SVA_PREFETCH_DISABLE BIT(26)
#define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30))
#define HZIP_SHAPER_RATE_COMPRESS 252
#define HZIP_SHAPER_RATE_DECOMPRESS 229
#define HZIP_DELAY_1_US 1
#define HZIP_POLL_TIMEOUT_US 1000
static const char hisi_zip_name[] = "hisi_zip";
static struct dentry *hzip_debugfs_root;
@ -262,6 +273,45 @@ int zip_create_qps(struct hisi_qp **qps, int qp_num, int node)
return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps);
}
static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm)
{
u32 val;
int ret;
if (qm->ver < QM_HW_V3)
return;
/* Enable prefetch */
val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);
val &= HZIP_PREFETCH_ENABLE;
writel(val, qm->io_base + HZIP_PREFETCH_CFG);
ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG,
val, !(val & HZIP_SVA_PREFETCH_DISABLE),
HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);
if (ret)
pci_err(qm->pdev, "failed to open sva prefetch\n");
}
static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm)
{
u32 val;
int ret;
if (qm->ver < QM_HW_V3)
return;
val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);
val |= HZIP_SVA_PREFETCH_DISABLE;
writel(val, qm->io_base + HZIP_PREFETCH_CFG);
ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS,
val, !(val & HZIP_SVA_DISABLE_READY),
HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);
if (ret)
pci_err(qm->pdev, "failed to close sva prefetch\n");
}
static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)
{
void __iomem *base = qm->io_base;
@ -312,10 +362,27 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)
return 0;
}
static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable)
{
u32 val1, val2;
val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
if (enable) {
val1 |= HZIP_AXI_SHUTDOWN_ENABLE;
val2 = HZIP_CORE_INT_RAS_NFE_ENABLE;
} else {
val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE;
val2 = 0x0;
}
if (qm->ver > QM_HW_V2)
writel(val2, qm->io_base + HZIP_OOO_SHUTDOWN_SEL);
writel(val1, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
}
static void hisi_zip_hw_error_enable(struct hisi_qm *qm)
{
u32 val;
if (qm->ver == QM_HW_V1) {
writel(HZIP_CORE_INT_MASK_ALL,
qm->io_base + HZIP_CORE_INT_MASK_REG);
@ -333,26 +400,20 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm)
writel(HZIP_CORE_INT_RAS_NFE_ENABLE,
qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
/* enable ZIP block master OOO when nfe occurs on Kunpeng930 */
hisi_zip_master_ooo_ctrl(qm, true);
/* enable ZIP hw error interrupts */
writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG);
/* enable ZIP block master OOO when m-bit error occur */
val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
val = val | HZIP_AXI_SHUTDOWN_ENABLE;
writel(val, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
}
static void hisi_zip_hw_error_disable(struct hisi_qm *qm)
{
u32 val;
/* disable ZIP hw error interrupts */
writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG);
/* disable ZIP block master OOO when m-bit error occur */
val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
val = val & ~HZIP_AXI_SHUTDOWN_ENABLE;
writel(val, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
/* disable ZIP block master OOO when nfe occurs on Kunpeng930 */
hisi_zip_master_ooo_ctrl(qm, false);
}
static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file)
@ -684,6 +745,8 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = {
.log_dev_hw_err = hisi_zip_log_hw_error,
.open_axi_master_ooo = hisi_zip_open_axi_master_ooo,
.close_axi_master_ooo = hisi_zip_close_axi_master_ooo,
.open_sva_prefetch = hisi_zip_open_sva_prefetch,
.close_sva_prefetch = hisi_zip_close_sva_prefetch,
.err_info_init = hisi_zip_err_info_init,
};
@ -702,6 +765,7 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)
qm->err_ini->err_info_init(qm);
hisi_zip_set_user_domain_and_cache(qm);
hisi_zip_open_sva_prefetch(qm);
hisi_qm_dev_err_init(qm);
hisi_zip_debug_regs_clear(qm);
@ -761,6 +825,7 @@ static void hisi_zip_qm_uninit(struct hisi_qm *qm)
static int hisi_zip_probe_init(struct hisi_zip *hisi_zip)
{
u32 type_rate = HZIP_SHAPER_RATE_COMPRESS;
struct hisi_qm *qm = &hisi_zip->qm;
int ret;
@ -768,6 +833,14 @@ static int hisi_zip_probe_init(struct hisi_zip *hisi_zip)
ret = hisi_zip_pf_probe_init(hisi_zip);
if (ret)
return ret;
/* enable shaper type 0 */
if (qm->ver >= QM_HW_V3) {
type_rate |= QM_SHAPER_ENABLE;
/* ZIP need to enable shaper type 1 */
type_rate |= HZIP_SHAPER_RATE_DECOMPRESS << QM_SHAPER_TYPE1_OFFSET;
qm->type_rate = type_rate;
}
}
return 0;

415
drivers/crypto/ixp4xx_crypto.c
View File

@ -15,6 +15,7 @@
#include <linux/spinlock.h>
#include <linux/gfp.h>
#include <linux/module.h>
#include <linux/of.h>
#include <crypto/ctr.h>
#include <crypto/internal/des.h>
@ -71,15 +72,11 @@
#define MOD_AES256 (0x0a00 | KEYLEN_256)
#define MAX_IVLEN 16
#define NPE_ID 2 /* NPE C */
#define NPE_QLEN 16
/* Space for registering when the first
* NPE_QLEN crypt_ctl are busy */
#define NPE_QLEN_TOTAL 64
#define SEND_QID 29
#define RECV_QID 30
#define CTL_FLAG_UNUSED 0x0000
#define CTL_FLAG_USED 0x1000
#define CTL_FLAG_PERFORM_ABLK 0x0001
@ -136,7 +133,7 @@ struct crypt_ctl {
u32 crypto_ctx; /* NPE Crypto Param structure address */
/* Used by Host: 4*4 bytes*/
unsigned ctl_flags;
unsigned int ctl_flags;
union {
struct skcipher_request *ablk_req;
struct aead_request *aead_req;
@ -149,6 +146,9 @@ struct crypt_ctl {
struct ablk_ctx {
struct buffer_desc *src;
struct buffer_desc *dst;
u8 iv[MAX_IVLEN];
bool encrypt;
struct skcipher_request fallback_req; // keep at the end
};
struct aead_ctx {
@ -181,9 +181,10 @@ struct ixp_ctx {
u8 enckey[MAX_KEYLEN];
u8 salt[MAX_IVLEN];
u8 nonce[CTR_RFC3686_NONCE_SIZE];
unsigned salted;
unsigned int salted;
atomic_t configuring;
struct completion completion;
struct crypto_skcipher *fallback_tfm;
};
struct ixp_alg {
@ -209,6 +210,7 @@ static const struct ix_hash_algo hash_alg_md5 = {
.icv = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
"\xFE\xDC\xBA\x98\x76\x54\x32\x10",
};
static const struct ix_hash_algo hash_alg_sha1 = {
.cfgword = 0x00000005,
.icv = "\x67\x45\x23\x01\xEF\xCD\xAB\x89\x98\xBA"
@ -216,16 +218,17 @@ static const struct ix_hash_algo hash_alg_sha1 = {
};
static struct npe *npe_c;
static struct dma_pool *buffer_pool = NULL;
static struct dma_pool *ctx_pool = NULL;
static struct crypt_ctl *crypt_virt = NULL;
static unsigned int send_qid;
static unsigned int recv_qid;
static struct dma_pool *buffer_pool;
static struct dma_pool *ctx_pool;
static struct crypt_ctl *crypt_virt;
static dma_addr_t crypt_phys;
static int support_aes = 1;
#define DRIVER_NAME "ixp4xx_crypto"
static struct platform_device *pdev;
static inline dma_addr_t crypt_virt2phys(struct crypt_ctl *virt)
@ -240,12 +243,12 @@ static inline struct crypt_ctl *crypt_phys2virt(dma_addr_t phys)
static inline u32 cipher_cfg_enc(struct crypto_tfm *tfm)
{
return container_of(tfm->__crt_alg, struct ixp_alg,crypto.base)->cfg_enc;
return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_enc;
}
static inline u32 cipher_cfg_dec(struct crypto_tfm *tfm)
{
return container_of(tfm->__crt_alg, struct ixp_alg,crypto.base)->cfg_dec;
return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_dec;
}
static inline const struct ix_hash_algo *ix_hash(struct crypto_tfm *tfm)
@ -256,6 +259,7 @@ static inline const struct ix_hash_algo *ix_hash(struct crypto_tfm *tfm)
static int setup_crypt_desc(void)
{
struct device *dev = &pdev->dev;
BUILD_BUG_ON(sizeof(struct crypt_ctl) != 64);
crypt_virt = dma_alloc_coherent(dev,
NPE_QLEN * sizeof(struct crypt_ctl),
@ -269,7 +273,7 @@ static DEFINE_SPINLOCK(desc_lock);
static struct crypt_ctl *get_crypt_desc(void)
{
int i;
static int idx = 0;
static int idx;
unsigned long flags;
spin_lock_irqsave(&desc_lock, flags);
@ -286,7 +290,7 @@ static struct crypt_ctl *get_crypt_desc(void)
idx = 0;
crypt_virt[i].ctl_flags = CTL_FLAG_USED;
spin_unlock_irqrestore(&desc_lock, flags);
return crypt_virt +i;
return crypt_virt + i;
} else {
spin_unlock_irqrestore(&desc_lock, flags);
return NULL;
@ -314,7 +318,7 @@ static struct crypt_ctl *get_crypt_desc_emerg(void)
idx = NPE_QLEN;
crypt_virt[i].ctl_flags = CTL_FLAG_USED;
spin_unlock_irqrestore(&emerg_lock, flags);
return crypt_virt +i;
return crypt_virt + i;
} else {
spin_unlock_irqrestore(&emerg_lock, flags);
return NULL;
@ -330,7 +334,7 @@ static void free_buf_chain(struct device *dev, struct buffer_desc *buf,
buf1 = buf->next;
phys1 = buf->phys_next;
dma_unmap_single(dev, buf->phys_next, buf->buf_len, buf->dir);
dma_unmap_single(dev, buf->phys_addr, buf->buf_len, buf->dir);
dma_pool_free(buffer_pool, buf, phys);
buf = buf1;
phys = phys1;
@ -348,8 +352,8 @@ static void finish_scattered_hmac(struct crypt_ctl *crypt)
int decryptlen = req->assoclen + req->cryptlen - authsize;
if (req_ctx->encrypt) {
scatterwalk_map_and_copy(req_ctx->hmac_virt,
req->dst, decryptlen, authsize, 1);
scatterwalk_map_and_copy(req_ctx->hmac_virt, req->dst,
decryptlen, authsize, 1);
}
dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes);
}
@ -372,19 +376,33 @@ static void one_packet(dma_addr_t phys)
free_buf_chain(dev, req_ctx->src, crypt->src_buf);
free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
if (req_ctx->hmac_virt) {
if (req_ctx->hmac_virt)
finish_scattered_hmac(crypt);
}
req->base.complete(&req->base, failed);
break;
}
case CTL_FLAG_PERFORM_ABLK: {
struct skcipher_request *req = crypt->data.ablk_req;
struct ablk_ctx *req_ctx = skcipher_request_ctx(req);
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
unsigned int ivsize = crypto_skcipher_ivsize(tfm);
unsigned int offset;
if (req_ctx->dst) {
free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
if (ivsize > 0) {
offset = req->cryptlen - ivsize;
if (req_ctx->encrypt) {
scatterwalk_map_and_copy(req->iv, req->dst,
offset, ivsize, 0);
} else {
memcpy(req->iv, req_ctx->iv, ivsize);
memzero_explicit(req_ctx->iv, ivsize);
}
}
if (req_ctx->dst)
free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
free_buf_chain(dev, req_ctx->src, crypt->src_buf);
req->base.complete(&req->base, failed);
break;
@ -392,14 +410,14 @@ static void one_packet(dma_addr_t phys)
case CTL_FLAG_GEN_ICV:
ctx = crypto_tfm_ctx(crypt->data.tfm);
dma_pool_free(ctx_pool, crypt->regist_ptr,
crypt->regist_buf->phys_addr);
crypt->regist_buf->phys_addr);
dma_pool_free(buffer_pool, crypt->regist_buf, crypt->src_buf);
if (atomic_dec_and_test(&ctx->configuring))
complete(&ctx->completion);
break;
case CTL_FLAG_GEN_REVAES:
ctx = crypto_tfm_ctx(crypt->data.tfm);
*(u32*)ctx->decrypt.npe_ctx &= cpu_to_be32(~CIPH_ENCR);
*(u32 *)ctx->decrypt.npe_ctx &= cpu_to_be32(~CIPH_ENCR);
if (atomic_dec_and_test(&ctx->configuring))
complete(&ctx->completion);
break;
@ -418,8 +436,8 @@ static void crypto_done_action(unsigned long arg)
{
int i;
for(i=0; i<4; i++) {
dma_addr_t phys = qmgr_get_entry(RECV_QID);
for (i = 0; i < 4; i++) {
dma_addr_t phys = qmgr_get_entry(recv_qid);
if (!phys)
return;
one_packet(phys);
@ -429,15 +447,52 @@ static void crypto_done_action(unsigned long arg)
static int init_ixp_crypto(struct device *dev)
{
int ret = -ENODEV;
struct device_node *np = dev->of_node;
u32 msg[2] = { 0, 0 };
int ret = -ENODEV;
u32 npe_id;
if (! ( ~(*IXP4XX_EXP_CFG2) & (IXP4XX_FEATURE_HASH |
IXP4XX_FEATURE_AES | IXP4XX_FEATURE_DES))) {
printk(KERN_ERR "ixp_crypto: No HW crypto available\n");
return ret;
dev_info(dev, "probing...\n");
/* Locate the NPE and queue manager to use from device tree */
if (IS_ENABLED(CONFIG_OF) && np) {
struct of_phandle_args queue_spec;
struct of_phandle_args npe_spec;
ret = of_parse_phandle_with_fixed_args(np, "intel,npe-handle",
1, 0, &npe_spec);
if (ret) {
dev_err(dev, "no NPE engine specified\n");
return -ENODEV;
}
npe_id = npe_spec.args[0];
ret = of_parse_phandle_with_fixed_args(np, "queue-rx", 1, 0,
&queue_spec);
if (ret) {
dev_err(dev, "no rx queue phandle\n");
return -ENODEV;
}
recv_qid = queue_spec.args[0];
ret = of_parse_phandle_with_fixed_args(np, "queue-txready", 1, 0,
&queue_spec);
if (ret) {
dev_err(dev, "no txready queue phandle\n");
return -ENODEV;
}
send_qid = queue_spec.args[0];
} else {
/*
* Hardcoded engine when using platform data, this goes away
* when we switch to using DT only.
*/
npe_id = 2;
send_qid = 29;
recv_qid = 30;
}
npe_c = npe_request(NPE_ID);
npe_c = npe_request(npe_id);
if (!npe_c)
return ret;
@ -455,10 +510,9 @@ static int init_ixp_crypto(struct device *dev)
goto npe_error;
}
switch ((msg[1]>>16) & 0xff) {
switch ((msg[1] >> 16) & 0xff) {
case 3:
printk(KERN_WARNING "Firmware of %s lacks AES support\n",
npe_name(npe_c));
dev_warn(dev, "Firmware of %s lacks AES support\n", npe_name(npe_c));
support_aes = 0;
break;
case 4:
@ -466,8 +520,7 @@ static int init_ixp_crypto(struct device *dev)
support_aes = 1;
break;
default:
printk(KERN_ERR "Firmware of %s lacks crypto support\n",
npe_name(npe_c));
dev_err(dev, "Firmware of %s lacks crypto support\n", npe_name(npe_c));
ret = -ENODEV;
goto npe_release;
}
@ -475,35 +528,34 @@ static int init_ixp_crypto(struct device *dev)
* so assure it is large enough
*/
BUILD_BUG_ON(SHA1_DIGEST_SIZE > sizeof(struct buffer_desc));
buffer_pool = dma_pool_create("buffer", dev,
sizeof(struct buffer_desc), 32, 0);
buffer_pool = dma_pool_create("buffer", dev, sizeof(struct buffer_desc),
32, 0);
ret = -ENOMEM;
if (!buffer_pool) {
if (!buffer_pool)
goto err;
}
ctx_pool = dma_pool_create("context", dev,
NPE_CTX_LEN, 16, 0);
if (!ctx_pool) {
ctx_pool = dma_pool_create("context", dev, NPE_CTX_LEN, 16, 0);
if (!ctx_pool)
goto err;
}
ret = qmgr_request_queue(SEND_QID, NPE_QLEN_TOTAL, 0, 0,
ret = qmgr_request_queue(send_qid, NPE_QLEN_TOTAL, 0, 0,
"ixp_crypto:out", NULL);
if (ret)
goto err;
ret = qmgr_request_queue(RECV_QID, NPE_QLEN, 0, 0,
ret = qmgr_request_queue(recv_qid, NPE_QLEN, 0, 0,
"ixp_crypto:in", NULL);
if (ret) {
qmgr_release_queue(SEND_QID);
qmgr_release_queue(send_qid);
goto err;
}
qmgr_set_irq(RECV_QID, QUEUE_IRQ_SRC_NOT_EMPTY, irqhandler, NULL);
qmgr_set_irq(recv_qid, QUEUE_IRQ_SRC_NOT_EMPTY, irqhandler, NULL);
tasklet_init(&crypto_done_tasklet, crypto_done_action, 0);
qmgr_enable_irq(RECV_QID);
qmgr_enable_irq(recv_qid);
return 0;
npe_error:
printk(KERN_ERR "%s not responding\n", npe_name(npe_c));
dev_err(dev, "%s not responding\n", npe_name(npe_c));
ret = -EIO;
err:
dma_pool_destroy(ctx_pool);
@ -515,22 +567,20 @@ static int init_ixp_crypto(struct device *dev)
static void release_ixp_crypto(struct device *dev)
{
qmgr_disable_irq(RECV_QID);
qmgr_disable_irq(recv_qid);
tasklet_kill(&crypto_done_tasklet);
qmgr_release_queue(SEND_QID);
qmgr_release_queue(RECV_QID);
qmgr_release_queue(send_qid);
qmgr_release_queue(recv_qid);
dma_pool_destroy(ctx_pool);
dma_pool_destroy(buffer_pool);
npe_release(npe_c);
if (crypt_virt) {
dma_free_coherent(dev,
NPE_QLEN * sizeof(struct crypt_ctl),
crypt_virt, crypt_phys);
}
if (crypt_virt)
dma_free_coherent(dev, NPE_QLEN * sizeof(struct crypt_ctl),
crypt_virt, crypt_phys);
}
static void reset_sa_dir(struct ix_sa_dir *dir)
@ -543,9 +593,9 @@ static void reset_sa_dir(struct ix_sa_dir *dir)
static int init_sa_dir(struct ix_sa_dir *dir)
{
dir->npe_ctx = dma_pool_alloc(ctx_pool, GFP_KERNEL, &dir->npe_ctx_phys);
if (!dir->npe_ctx) {
if (!dir->npe_ctx)
return -ENOMEM;
}
reset_sa_dir(dir);
return 0;
}
@ -566,15 +616,31 @@ static int init_tfm(struct crypto_tfm *tfm)
if (ret)
return ret;
ret = init_sa_dir(&ctx->decrypt);
if (ret) {
if (ret)
free_sa_dir(&ctx->encrypt);
}
return ret;
}
static int init_tfm_ablk(struct crypto_skcipher *tfm)
{
crypto_skcipher_set_reqsize(tfm, sizeof(struct ablk_ctx));
struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm);
struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm);
const char *name = crypto_tfm_alg_name(ctfm);
ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(ctx->fallback_tfm)) {
pr_err("ERROR: Cannot allocate fallback for %s %ld\n",
name, PTR_ERR(ctx->fallback_tfm));
return PTR_ERR(ctx->fallback_tfm);
}
pr_info("Fallback for %s is %s\n",
crypto_tfm_alg_driver_name(&tfm->base),
crypto_tfm_alg_driver_name(crypto_skcipher_tfm(ctx->fallback_tfm))
);
crypto_skcipher_set_reqsize(tfm, sizeof(struct ablk_ctx) + crypto_skcipher_reqsize(ctx->fallback_tfm));
return init_tfm(crypto_skcipher_tfm(tfm));
}
@ -587,12 +653,17 @@ static int init_tfm_aead(struct crypto_aead *tfm)
static void exit_tfm(struct crypto_tfm *tfm)
{
struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
free_sa_dir(&ctx->encrypt);
free_sa_dir(&ctx->decrypt);
}
static void exit_tfm_ablk(struct crypto_skcipher *tfm)
{
struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm);
struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm);
crypto_free_skcipher(ctx->fallback_tfm);
exit_tfm(crypto_skcipher_tfm(tfm));
}
@ -602,7 +673,8 @@ static void exit_tfm_aead(struct crypto_aead *tfm)
}
static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target,
int init_len, u32 ctx_addr, const u8 *key, int key_len)
int init_len, u32 ctx_addr, const u8 *key,
int key_len)
{
struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypt_ctl *crypt;
@ -629,9 +701,8 @@ static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target,
memcpy(pad, key, key_len);
memset(pad + key_len, 0, HMAC_PAD_BLOCKLEN - key_len);
for (i = 0; i < HMAC_PAD_BLOCKLEN; i++) {
for (i = 0; i < HMAC_PAD_BLOCKLEN; i++)
pad[i] ^= xpad;
}
crypt->data.tfm = tfm;
crypt->regist_ptr = pad;
@ -652,13 +723,13 @@ static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target,
buf->phys_addr = pad_phys;
atomic_inc(&ctx->configuring);
qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
BUG_ON(qmgr_stat_overflow(SEND_QID));
qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
BUG_ON(qmgr_stat_overflow(send_qid));
return 0;
}
static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned authsize,
const u8 *key, int key_len, unsigned digest_len)
static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned int authsize,
const u8 *key, int key_len, unsigned int digest_len)
{
u32 itarget, otarget, npe_ctx_addr;
unsigned char *cinfo;
@ -673,11 +744,11 @@ static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned authsize,
algo = ix_hash(tfm);
/* write cfg word to cryptinfo */
cfgword = algo->cfgword | ( authsize << 6); /* (authsize/4) << 8 */
cfgword = algo->cfgword | (authsize << 6); /* (authsize/4) << 8 */
#ifndef __ARMEB__
cfgword ^= 0xAA000000; /* change the "byte swap" flags */
#endif
*(u32*)cinfo = cpu_to_be32(cfgword);
*(u32 *)cinfo = cpu_to_be32(cfgword);
cinfo += sizeof(cfgword);
/* write ICV to cryptinfo */
@ -697,11 +768,11 @@ static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned authsize,
dir->npe_mode |= NPE_OP_HASH_VERIFY;
ret = register_chain_var(tfm, HMAC_OPAD_VALUE, otarget,
init_len, npe_ctx_addr, key, key_len);
init_len, npe_ctx_addr, key, key_len);
if (ret)
return ret;
return register_chain_var(tfm, HMAC_IPAD_VALUE, itarget,
init_len, npe_ctx_addr, key, key_len);
init_len, npe_ctx_addr, key, key_len);
}
static int gen_rev_aes_key(struct crypto_tfm *tfm)
@ -711,10 +782,10 @@ static int gen_rev_aes_key(struct crypto_tfm *tfm)
struct ix_sa_dir *dir = &ctx->decrypt;
crypt = get_crypt_desc_emerg();
if (!crypt) {
if (!crypt)
return -EAGAIN;
}
*(u32*)dir->npe_ctx |= cpu_to_be32(CIPH_ENCR);
*(u32 *)dir->npe_ctx |= cpu_to_be32(CIPH_ENCR);
crypt->data.tfm = tfm;
crypt->crypt_offs = 0;
@ -727,13 +798,13 @@ static int gen_rev_aes_key(struct crypto_tfm *tfm)
crypt->ctl_flags |= CTL_FLAG_GEN_REVAES;
atomic_inc(&ctx->configuring);
qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
BUG_ON(qmgr_stat_overflow(SEND_QID));
qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
BUG_ON(qmgr_stat_overflow(send_qid));
return 0;
}
static int setup_cipher(struct crypto_tfm *tfm, int encrypt,
const u8 *key, int key_len)
static int setup_cipher(struct crypto_tfm *tfm, int encrypt, const u8 *key,
int key_len)
{
u8 *cinfo;
u32 cipher_cfg;
@ -753,9 +824,15 @@ static int setup_cipher(struct crypto_tfm *tfm, int encrypt,
}
if (cipher_cfg & MOD_AES) {
switch (key_len) {
case 16: keylen_cfg = MOD_AES128; break;
case 24: keylen_cfg = MOD_AES192; break;
case 32: keylen_cfg = MOD_AES256; break;
case 16:
keylen_cfg = MOD_AES128;
break;
case 24:
keylen_cfg = MOD_AES192;
break;
case 32:
keylen_cfg = MOD_AES256;
break;
default:
return -EINVAL;
}
@ -766,31 +843,31 @@ static int setup_cipher(struct crypto_tfm *tfm, int encrypt,
return err;
}
/* write cfg word to cryptinfo */
*(u32*)cinfo = cpu_to_be32(cipher_cfg);
*(u32 *)cinfo = cpu_to_be32(cipher_cfg);
cinfo += sizeof(cipher_cfg);
/* write cipher key to cryptinfo */
memcpy(cinfo, key, key_len);
/* NPE wants keylen set to DES3_EDE_KEY_SIZE even for single DES */
if (key_len < DES3_EDE_KEY_SIZE && !(cipher_cfg & MOD_AES)) {
memset(cinfo + key_len, 0, DES3_EDE_KEY_SIZE -key_len);
memset(cinfo + key_len, 0, DES3_EDE_KEY_SIZE - key_len);
key_len = DES3_EDE_KEY_SIZE;
}
dir->npe_ctx_idx = sizeof(cipher_cfg) + key_len;
dir->npe_mode |= NPE_OP_CRYPT_ENABLE;
if ((cipher_cfg & MOD_AES) && !encrypt) {
if ((cipher_cfg & MOD_AES) && !encrypt)
return gen_rev_aes_key(tfm);
}
return 0;
}
static struct buffer_desc *chainup_buffers(struct device *dev,
struct scatterlist *sg, unsigned nbytes,
struct scatterlist *sg, unsigned int nbytes,
struct buffer_desc *buf, gfp_t flags,
enum dma_data_direction dir)
{
for (; nbytes > 0; sg = sg_next(sg)) {
unsigned len = min(nbytes, sg->length);
unsigned int len = min(nbytes, sg->length);
struct buffer_desc *next_buf;
dma_addr_t next_buf_phys;
void *ptr;
@ -817,7 +894,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev,
}
static int ablk_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int key_len)
unsigned int key_len)
{
struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
int ret;
@ -838,7 +915,12 @@ static int ablk_setkey(struct crypto_skcipher *tfm, const u8 *key,
out:
if (!atomic_dec_and_test(&ctx->configuring))
wait_for_completion(&ctx->completion);
return ret;
if (ret)
return ret;
crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK);
crypto_skcipher_set_flags(ctx->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
return crypto_skcipher_setkey(ctx->fallback_tfm, key, key_len);
}
static int ablk_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
@ -849,7 +931,7 @@ static int ablk_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
}
static int ablk_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int key_len)
unsigned int key_len)
{
struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
@ -858,17 +940,36 @@ static int ablk_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key,
return -EINVAL;
memcpy(ctx->nonce, key + (key_len - CTR_RFC3686_NONCE_SIZE),
CTR_RFC3686_NONCE_SIZE);
CTR_RFC3686_NONCE_SIZE);
key_len -= CTR_RFC3686_NONCE_SIZE;
return ablk_setkey(tfm, key, key_len);
}
static int ixp4xx_cipher_fallback(struct skcipher_request *areq, int encrypt)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
struct ixp_ctx *op = crypto_skcipher_ctx(tfm);
struct ablk_ctx *rctx = skcipher_request_ctx(areq);
int err;
skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
areq->base.complete, areq->base.data);
skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
areq->cryptlen, areq->iv);
if (encrypt)
err = crypto_skcipher_encrypt(&rctx->fallback_req);
else
err = crypto_skcipher_decrypt(&rctx->fallback_req);
return err;
}
static int ablk_perform(struct skcipher_request *req, int encrypt)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
unsigned ivsize = crypto_skcipher_ivsize(tfm);
unsigned int ivsize = crypto_skcipher_ivsize(tfm);
struct ix_sa_dir *dir;
struct crypt_ctl *crypt;
unsigned int nbytes = req->cryptlen;
@ -876,15 +977,20 @@ static int ablk_perform(struct skcipher_request *req, int encrypt)
struct ablk_ctx *req_ctx = skcipher_request_ctx(req);
struct buffer_desc src_hook;
struct device *dev = &pdev->dev;
unsigned int offset;
gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
GFP_KERNEL : GFP_ATOMIC;
if (qmgr_stat_full(SEND_QID))
if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1)
return ixp4xx_cipher_fallback(req, encrypt);
if (qmgr_stat_full(send_qid))
return -EAGAIN;
if (atomic_read(&ctx->configuring))
return -EAGAIN;
dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
req_ctx->encrypt = encrypt;
crypt = get_crypt_desc();
if (!crypt)
@ -900,14 +1006,19 @@ static int ablk_perform(struct skcipher_request *req, int encrypt)
BUG_ON(ivsize && !req->iv);
memcpy(crypt->iv, req->iv, ivsize);
if (ivsize > 0 && !encrypt) {
offset = req->cryptlen - ivsize;
scatterwalk_map_and_copy(req_ctx->iv, req->src, offset, ivsize, 0);
}
if (req->src != req->dst) {
struct buffer_desc dst_hook;
crypt->mode |= NPE_OP_NOT_IN_PLACE;
/* This was never tested by Intel
* for more than one dst buffer, I think. */
req_ctx->dst = NULL;
if (!chainup_buffers(dev, req->dst, nbytes, &dst_hook,
flags, DMA_FROM_DEVICE))
flags, DMA_FROM_DEVICE))
goto free_buf_dest;
src_direction = DMA_TO_DEVICE;
req_ctx->dst = dst_hook.next;
@ -916,23 +1027,23 @@ static int ablk_perform(struct skcipher_request *req, int encrypt)
req_ctx->dst = NULL;
}
req_ctx->src = NULL;
if (!chainup_buffers(dev, req->src, nbytes, &src_hook,
flags, src_direction))
if (!chainup_buffers(dev, req->src, nbytes, &src_hook, flags,
src_direction))
goto free_buf_src;
req_ctx->src = src_hook.next;
crypt->src_buf = src_hook.phys_next;
crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK;
qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
BUG_ON(qmgr_stat_overflow(SEND_QID));
qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
BUG_ON(qmgr_stat_overflow(send_qid));
return -EINPROGRESS;
free_buf_src:
free_buf_chain(dev, req_ctx->src, crypt->src_buf);
free_buf_dest:
if (req->src != req->dst) {
if (req->src != req->dst)
free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
}
crypt->ctl_flags = CTL_FLAG_UNUSED;
return -ENOMEM;
}
@ -956,7 +1067,7 @@ static int ablk_rfc3686_crypt(struct skcipher_request *req)
int ret;
/* set up counter block */
memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
memcpy(iv + CTR_RFC3686_NONCE_SIZE, info, CTR_RFC3686_IV_SIZE);
/* initialize counter portion of counter block */
@ -970,12 +1081,12 @@ static int ablk_rfc3686_crypt(struct skcipher_request *req)
}
static int aead_perform(struct aead_request *req, int encrypt,
int cryptoffset, int eff_cryptlen, u8 *iv)
int cryptoffset, int eff_cryptlen, u8 *iv)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
unsigned ivsize = crypto_aead_ivsize(tfm);
unsigned authsize = crypto_aead_authsize(tfm);
unsigned int ivsize = crypto_aead_ivsize(tfm);
unsigned int authsize = crypto_aead_authsize(tfm);
struct ix_sa_dir *dir;
struct crypt_ctl *crypt;
unsigned int cryptlen;
@ -987,7 +1098,7 @@ static int aead_perform(struct aead_request *req, int encrypt,
enum dma_data_direction src_direction = DMA_BIDIRECTIONAL;
unsigned int lastlen;
if (qmgr_stat_full(SEND_QID))
if (qmgr_stat_full(send_qid))
return -EAGAIN;
if (atomic_read(&ctx->configuring))
return -EAGAIN;
@ -998,7 +1109,7 @@ static int aead_perform(struct aead_request *req, int encrypt,
} else {
dir = &ctx->decrypt;
/* req->cryptlen includes the authsize when decrypting */
cryptlen = req->cryptlen -authsize;
cryptlen = req->cryptlen - authsize;
eff_cryptlen -= authsize;
}
crypt = get_crypt_desc();
@ -1058,12 +1169,12 @@ static int aead_perform(struct aead_request *req, int encrypt,
/* The 12 hmac bytes are scattered,
* we need to copy them into a safe buffer */
req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags,
&crypt->icv_rev_aes);
&crypt->icv_rev_aes);
if (unlikely(!req_ctx->hmac_virt))
goto free_buf_dst;
if (!encrypt) {
scatterwalk_map_and_copy(req_ctx->hmac_virt,
req->src, cryptlen, authsize, 0);
req->src, cryptlen, authsize, 0);
}
req_ctx->encrypt = encrypt;
} else {
@ -1071,8 +1182,8 @@ static int aead_perform(struct aead_request *req, int encrypt,
}
crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD;
qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
BUG_ON(qmgr_stat_overflow(SEND_QID));
qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
BUG_ON(qmgr_stat_overflow(send_qid));
return -EINPROGRESS;
free_buf_dst:
@ -1086,7 +1197,7 @@ static int aead_perform(struct aead_request *req, int encrypt,
static int aead_setup(struct crypto_aead *tfm, unsigned int authsize)
{
struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
unsigned digest_len = crypto_aead_maxauthsize(tfm);
unsigned int digest_len = crypto_aead_maxauthsize(tfm);
int ret;
if (!ctx->enckey_len && !ctx->authkey_len)
@ -1104,11 +1215,11 @@ static int aead_setup(struct crypto_aead *tfm, unsigned int authsize)
if (ret)
goto out;
ret = setup_auth(&tfm->base, 0, authsize, ctx->authkey,
ctx->authkey_len, digest_len);
ctx->authkey_len, digest_len);
if (ret)
goto out;
ret = setup_auth(&tfm->base, 1, authsize, ctx->authkey,
ctx->authkey_len, digest_len);
ctx->authkey_len, digest_len);
out:
if (!atomic_dec_and_test(&ctx->configuring))
wait_for_completion(&ctx->completion);
@ -1119,13 +1230,13 @@ static int aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
int max = crypto_aead_maxauthsize(tfm) >> 2;
if ((authsize>>2) < 1 || (authsize>>2) > max || (authsize & 3))
if ((authsize >> 2) < 1 || (authsize >> 2) > max || (authsize & 3))
return -EINVAL;
return aead_setup(tfm, authsize);
}
static int aead_setkey(struct crypto_aead *tfm, const u8 *key,
unsigned int keylen)
unsigned int keylen)
{
struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
struct crypto_authenc_keys keys;
@ -1364,43 +1475,33 @@ static struct ixp_aead_alg ixp4xx_aeads[] = {
#define IXP_POSTFIX "-ixp4xx"
static const struct platform_device_info ixp_dev_info __initdata = {
.name = DRIVER_NAME,
.id = 0,
.dma_mask = DMA_BIT_MASK(32),
};
static int __init ixp_module_init(void)
static int ixp_crypto_probe(struct platform_device *_pdev)
{
struct device *dev = &_pdev->dev;
int num = ARRAY_SIZE(ixp4xx_algos);
int i, err;
pdev = platform_device_register_full(&ixp_dev_info);
if (IS_ERR(pdev))
return PTR_ERR(pdev);
pdev = _pdev;
err = init_ixp_crypto(&pdev->dev);
if (err) {
platform_device_unregister(pdev);
err = init_ixp_crypto(dev);
if (err)
return err;
}
for (i=0; i< num; i++) {
for (i = 0; i < num; i++) {
struct skcipher_alg *cra = &ixp4xx_algos[i].crypto;
if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
"%s"IXP_POSTFIX, cra->base.cra_name) >=
CRYPTO_MAX_ALG_NAME)
{
"%s"IXP_POSTFIX, cra->base.cra_name) >=
CRYPTO_MAX_ALG_NAME)
continue;
}
if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) {
if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES))
continue;
}
/* block ciphers */
cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_ALLOCATES_MEMORY;
CRYPTO_ALG_ALLOCATES_MEMORY |
CRYPTO_ALG_NEED_FALLBACK;
if (!cra->setkey)
cra->setkey = ablk_setkey;
if (!cra->encrypt)
@ -1415,7 +1516,7 @@ static int __init ixp_module_init(void)
cra->base.cra_alignmask = 3;
cra->base.cra_priority = 300;
if (crypto_register_skcipher(cra))
printk(KERN_ERR "Failed to register '%s'\n",
dev_err(&pdev->dev, "Failed to register '%s'\n",
cra->base.cra_name);
else
ixp4xx_algos[i].registered = 1;
@ -1448,7 +1549,7 @@ static int __init ixp_module_init(void)
cra->base.cra_priority = 300;
if (crypto_register_aead(cra))
printk(KERN_ERR "Failed to register '%s'\n",
dev_err(&pdev->dev, "Failed to register '%s'\n",
cra->base.cra_driver_name);
else
ixp4xx_aeads[i].registered = 1;
@ -1456,7 +1557,7 @@ static int __init ixp_module_init(void)
return 0;
}
static void __exit ixp_module_exit(void)
static int ixp_crypto_remove(struct platform_device *pdev)
{
int num = ARRAY_SIZE(ixp4xx_algos);
int i;
@ -1466,16 +1567,30 @@ static void __exit ixp_module_exit(void)
crypto_unregister_aead(&ixp4xx_aeads[i].crypto);
}
for (i=0; i< num; i++) {
for (i = 0; i < num; i++) {
if (ixp4xx_algos[i].registered)
crypto_unregister_skcipher(&ixp4xx_algos[i].crypto);
}
release_ixp_crypto(&pdev->dev);
platform_device_unregister(pdev);
}
module_init(ixp_module_init);
module_exit(ixp_module_exit);
return 0;
}
static const struct of_device_id ixp4xx_crypto_of_match[] = {
{
.compatible = "intel,ixp4xx-crypto",
},
{},
};
static struct platform_driver ixp_crypto_driver = {
.probe = ixp_crypto_probe,
.remove = ixp_crypto_remove,
.driver = {
.name = "ixp4xx_crypto",
.of_match_table = ixp4xx_crypto_of_match,
},
};
module_platform_driver(ixp_crypto_driver);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Christian Hohnstaedt <chohnstaedt@innominate.com>");

2
drivers/crypto/marvell/cesa/cesa.h
View File

@ -66,7 +66,7 @@
#define CESA_SA_ST_ACT_1 BIT(1)
/*
* CESA_SA_FPGA_INT_STATUS looks like a FPGA leftover and is documented only
* CESA_SA_FPGA_INT_STATUS looks like an FPGA leftover and is documented only
* in Errata 4.12. It looks like that it was part of an IRQ-controller in FPGA
* and someone forgot to remove it while switching to the core and moving to
* CESA_SA_INT_STATUS.

13
drivers/crypto/marvell/octeontx2/Makefile
View File

@ -1,10 +1,11 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += octeontx2-cpt.o octeontx2-cptvf.o
obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += rvu_cptpf.o rvu_cptvf.o
octeontx2-cpt-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \
otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o
octeontx2-cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \
otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \
otx2_cptvf_algs.o
rvu_cptpf-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \
otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o \
cn10k_cpt.o
rvu_cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \
otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \
otx2_cptvf_algs.o cn10k_cpt.o
ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af

93
drivers/crypto/marvell/octeontx2/cn10k_cpt.c Normal file
View File

@ -0,0 +1,93 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2021 Marvell. */
#include <linux/soc/marvell/octeontx2/asm.h>
#include "otx2_cptpf.h"
#include "otx2_cptvf.h"
#include "otx2_cptlf.h"
#include "cn10k_cpt.h"
static struct cpt_hw_ops otx2_hw_ops = {
.send_cmd = otx2_cpt_send_cmd,
.cpt_get_compcode = otx2_cpt_get_compcode,
.cpt_get_uc_compcode = otx2_cpt_get_uc_compcode,
};
static struct cpt_hw_ops cn10k_hw_ops = {
.send_cmd = cn10k_cpt_send_cmd,
.cpt_get_compcode = cn10k_cpt_get_compcode,
.cpt_get_uc_compcode = cn10k_cpt_get_uc_compcode,
};
void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num,
struct otx2_cptlf_info *lf)
{
void __iomem *lmtline = lf->lmtline;
u64 val = (lf->slot & 0x7FF);
u64 tar_addr = 0;
/* tar_addr<6:4> = Size of first LMTST - 1 in units of 128b. */
tar_addr |= (__force u64)lf->ioreg |
(((OTX2_CPT_INST_SIZE/16) - 1) & 0x7) << 4;
/*
* Make sure memory areas pointed in CPT_INST_S
* are flushed before the instruction is sent to CPT
*/
dma_wmb();
/* Copy CPT command to LMTLINE */
memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE);
cn10k_lmt_flush(val, tar_addr);
}
int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf)
{
struct pci_dev *pdev = cptpf->pdev;
resource_size_t size;
u64 lmt_base;
if (!test_bit(CN10K_LMTST, &cptpf->cap_flag)) {
cptpf->lfs.ops = &otx2_hw_ops;
return 0;
}
cptpf->lfs.ops = &cn10k_hw_ops;
lmt_base = readq(cptpf->reg_base + RVU_PF_LMTLINE_ADDR);
if (!lmt_base) {
dev_err(&pdev->dev, "PF LMTLINE address not configured\n");
return -ENOMEM;
}
size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM);
size -= ((1 + cptpf->max_vfs) * MBOX_SIZE);
cptpf->lfs.lmt_base = devm_ioremap_wc(&pdev->dev, lmt_base, size);
if (!cptpf->lfs.lmt_base) {
dev_err(&pdev->dev,
"Mapping of PF LMTLINE address failed\n");
return -ENOMEM;
}
return 0;
}
int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
resource_size_t offset, size;
if (!test_bit(CN10K_LMTST, &cptvf->cap_flag)) {
cptvf->lfs.ops = &otx2_hw_ops;
return 0;
}
cptvf->lfs.ops = &cn10k_hw_ops;
offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM);
size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM);
/* Map VF LMILINE region */
cptvf->lfs.lmt_base = devm_ioremap_wc(&pdev->dev, offset, size);
if (!cptvf->lfs.lmt_base) {
dev_err(&pdev->dev, "Unable to map BAR4\n");
return -ENOMEM;
}
return 0;
}

36
drivers/crypto/marvell/octeontx2/cn10k_cpt.h Normal file
View File

@ -0,0 +1,36 @@
/* SPDX-License-Identifier: GPL-2.0-only
* Copyright (C) 2021 Marvell.
*/
#ifndef __CN10K_CPT_H
#define __CN10K_CPT_H
#include "otx2_cpt_common.h"
#include "otx2_cptpf.h"
#include "otx2_cptvf.h"
static inline u8 cn10k_cpt_get_compcode(union otx2_cpt_res_s *result)
{
return ((struct cn10k_cpt_res_s *)result)->compcode;
}
static inline u8 cn10k_cpt_get_uc_compcode(union otx2_cpt_res_s *result)
{
return ((struct cn10k_cpt_res_s *)result)->uc_compcode;
}
static inline u8 otx2_cpt_get_compcode(union otx2_cpt_res_s *result)
{
return ((struct cn9k_cpt_res_s *)result)->compcode;
}
static inline u8 otx2_cpt_get_uc_compcode(union otx2_cpt_res_s *result)
{
return ((struct cn9k_cpt_res_s *)result)->uc_compcode;
}
void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num,
struct otx2_cptlf_info *lf);
int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf);
int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf);
#endif /* __CN10K_CPTLF_H */

23
drivers/crypto/marvell/octeontx2/otx2_cpt_common.h
View File

@ -25,6 +25,10 @@
#define OTX2_CPT_NAME_LENGTH 64
#define OTX2_CPT_DMA_MINALIGN 128
/* HW capability flags */
#define CN10K_MBOX 0
#define CN10K_LMTST 1
#define BAD_OTX2_CPT_ENG_TYPE OTX2_CPT_MAX_ENG_TYPES
enum otx2_cpt_eng_type {
@ -116,6 +120,25 @@ static inline u64 otx2_cpt_read64(void __iomem *reg_base, u64 blk, u64 slot,
OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs));
}
static inline bool is_dev_otx2(struct pci_dev *pdev)
{
if (pdev->device == OTX2_CPT_PCI_PF_DEVICE_ID ||
pdev->device == OTX2_CPT_PCI_VF_DEVICE_ID)
return true;
return false;
}
static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev,
unsigned long *cap_flag)
{
if (!is_dev_otx2(pdev)) {
__set_bit(CN10K_MBOX, cap_flag);
__set_bit(CN10K_LMTST, cap_flag);
}
}
int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev);
int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev);

16
drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h
View File

@ -10,6 +10,8 @@
/* Device IDs */
#define OTX2_CPT_PCI_PF_DEVICE_ID 0xA0FD
#define OTX2_CPT_PCI_VF_DEVICE_ID 0xA0FE
#define CN10K_CPT_PCI_PF_DEVICE_ID 0xA0F2
#define CN10K_CPT_PCI_VF_DEVICE_ID 0xA0F3
/* Mailbox interrupts offset */
#define OTX2_CPT_PF_MBOX_INT 6
@ -25,6 +27,7 @@
*/
#define OTX2_CPT_VF_MSIX_VECTORS 1
#define OTX2_CPT_VF_INTR_MBOX_MASK BIT(0)
#define CN10K_CPT_VF_MBOX_REGION (0xC0000)
/* CPT LF MSIX vectors */
#define OTX2_CPT_LF_MSIX_VECTORS 2
@ -135,7 +138,7 @@ enum otx2_cpt_comp_e {
OTX2_CPT_COMP_E_FAULT = 0x02,
OTX2_CPT_COMP_E_HWERR = 0x04,
OTX2_CPT_COMP_E_INSTERR = 0x05,
OTX2_CPT_COMP_E_LAST_ENTRY = 0x06
OTX2_CPT_COMP_E_WARN = 0x06
};
/*
@ -266,13 +269,22 @@ union otx2_cpt_inst_s {
union otx2_cpt_res_s {
u64 u[2];
struct {
struct cn9k_cpt_res_s {
u64 compcode:8;
u64 uc_compcode:8;
u64 doneint:1;
u64 reserved_17_63:47;
u64 reserved_64_127;
} s;
struct cn10k_cpt_res_s {
u64 compcode:7;
u64 doneint:1;
u64 uc_compcode:8;
u64 rlen:16;
u64 spi:32;
u64 esn;
} cn10k;
};
/*

9
drivers/crypto/marvell/octeontx2/otx2_cptlf.c
View File

@ -379,9 +379,14 @@ int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri,
for (slot = 0; slot < lfs->lfs_num; slot++) {
lfs->lf[slot].lfs = lfs;
lfs->lf[slot].slot = slot;
lfs->lf[slot].lmtline = lfs->reg_base +
OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot,
if (lfs->lmt_base)
lfs->lf[slot].lmtline = lfs->lmt_base +
(slot * LMTLINE_SIZE);
else
lfs->lf[slot].lmtline = lfs->reg_base +
OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot,
OTX2_CPT_LMT_LF_LMTLINEX(0));
lfs->lf[slot].ioreg = lfs->reg_base +
OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_CPT0, slot,
OTX2_CPT_LF_NQX(0));

10
drivers/crypto/marvell/octeontx2/otx2_cptlf.h
View File

@ -84,12 +84,22 @@ struct otx2_cptlf_info {
struct otx2_cptlf_wqe *wqe; /* Tasklet work info */
};
struct cpt_hw_ops {
void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num,
struct otx2_cptlf_info *lf);
u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result);
u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result);
};
struct otx2_cptlfs_info {
/* Registers start address of VF/PF LFs are attached to */
void __iomem *reg_base;
#define LMTLINE_SIZE 128
void __iomem *lmt_base;
struct pci_dev *pdev; /* Device LFs are attached to */
struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM];
struct otx2_mbox *mbox;
struct cpt_hw_ops *ops;
u8 are_lfs_attached; /* Whether CPT LFs are attached */
u8 lfs_num; /* Number of CPT LFs */
u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */

1
drivers/crypto/marvell/octeontx2/otx2_cptpf.h
View File

@ -47,6 +47,7 @@ struct otx2_cptpf_dev {
struct workqueue_struct *flr_wq;
struct cptpf_flr_work *flr_work;
unsigned long cap_flag;
u8 pf_id; /* RVU PF number */
u8 max_vfs; /* Maximum number of VFs supported by CPT */
u8 enabled_vfs; /* Number of enabled VFs */

164
drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
View File

@ -6,10 +6,11 @@
#include "otx2_cpt_common.h"
#include "otx2_cptpf_ucode.h"
#include "otx2_cptpf.h"
#include "cn10k_cpt.h"
#include "rvu_reg.h"
#define OTX2_CPT_DRV_NAME "octeontx2-cpt"
#define OTX2_CPT_DRV_STRING "Marvell OcteonTX2 CPT Physical Function Driver"
#define OTX2_CPT_DRV_NAME "rvu_cptpf"
#define OTX2_CPT_DRV_STRING "Marvell RVU CPT Physical Function Driver"
static void cptpf_enable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf,
int num_vfs)
@ -62,45 +63,66 @@ static void cptpf_disable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf,
}
}
static void cptpf_enable_vf_flr_intrs(struct otx2_cptpf_dev *cptpf)
static void cptpf_enable_vf_flr_me_intrs(struct otx2_cptpf_dev *cptpf,
int num_vfs)
{
/* Clear interrupt if any */
/* Clear FLR interrupt if any */
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0),
~0x0ULL);
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1),
~0x0ULL);
INTR_MASK(num_vfs));
/* Enable VF FLR interrupts */
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFFLR_INT_ENA_W1SX(0), ~0x0ULL);
RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(num_vfs));
/* Clear ME interrupt if any */
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFME_INTX(0),
INTR_MASK(num_vfs));
/* Enable VF ME interrupts */
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFFLR_INT_ENA_W1SX(1), ~0x0ULL);
RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(num_vfs));
if (num_vfs <= 64)
return;
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1),
INTR_MASK(num_vfs - 64));
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(num_vfs - 64));
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFME_INTX(1),
INTR_MASK(num_vfs - 64));
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(num_vfs - 64));
}
static void cptpf_disable_vf_flr_intrs(struct otx2_cptpf_dev *cptpf,
static void cptpf_disable_vf_flr_me_intrs(struct otx2_cptpf_dev *cptpf,
int num_vfs)
{
int vector;
/* Disable VF FLR interrupts */
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFFLR_INT_ENA_W1CX(0), ~0x0ULL);
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFFLR_INT_ENA_W1CX(1), ~0x0ULL);
/* Clear interrupt if any */
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0),
~0x0ULL);
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1),
~0x0ULL);
RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(num_vfs));
vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR0);
free_irq(vector, cptpf);
if (num_vfs > 64) {
vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR1);
free_irq(vector, cptpf);
}
/* Disable VF ME interrupts */
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(num_vfs));
vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFME0);
free_irq(vector, cptpf);
if (num_vfs <= 64)
return;
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(num_vfs - 64));
vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR1);
free_irq(vector, cptpf);
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(num_vfs - 64));
vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFME1);
free_irq(vector, cptpf);
}
static void cptpf_flr_wq_handler(struct work_struct *work)
@ -172,11 +194,38 @@ static irqreturn_t cptpf_vf_flr_intr(int __always_unused irq, void *arg)
return IRQ_HANDLED;
}
static irqreturn_t cptpf_vf_me_intr(int __always_unused irq, void *arg)
{
struct otx2_cptpf_dev *cptpf = arg;
int reg, vf, num_reg = 1;
u64 intr;
if (cptpf->max_vfs > 64)
num_reg = 2;
for (reg = 0; reg < num_reg; reg++) {
intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFME_INTX(reg));
if (!intr)
continue;
for (vf = 0; vf < 64; vf++) {
if (!(intr & BIT_ULL(vf)))
continue;
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
/* Clear interrupt */
otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0,
RVU_PF_VFME_INTX(reg), BIT_ULL(vf));
}
}
return IRQ_HANDLED;
}
static void cptpf_unregister_vfpf_intr(struct otx2_cptpf_dev *cptpf,
int num_vfs)
{
cptpf_disable_vfpf_mbox_intr(cptpf, num_vfs);
cptpf_disable_vf_flr_intrs(cptpf, num_vfs);
cptpf_disable_vf_flr_me_intrs(cptpf, num_vfs);
}
static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs)
@ -202,6 +251,15 @@ static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs)
"IRQ registration failed for VFFLR0 irq\n");
goto free_mbox0_irq;
}
vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME0);
/* Register VF ME interrupt handler */
ret = request_irq(vector, cptpf_vf_me_intr, 0, "CPTPF ME0", cptpf);
if (ret) {
dev_err(dev,
"IRQ registration failed for PFVF mbox0 irq\n");
goto free_flr0_irq;
}
if (num_vfs > 64) {
vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0,
@ -209,7 +267,7 @@ static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs)
if (ret) {
dev_err(dev,
"IRQ registration failed for PFVF mbox1 irq\n");
goto free_flr0_irq;
goto free_me0_irq;
}
vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1);
/* Register VF FLR interrupt handler */
@ -220,15 +278,30 @@ static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs)
"IRQ registration failed for VFFLR1 irq\n");
goto free_mbox1_irq;
}
vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME1);
/* Register VF FLR interrupt handler */
ret = request_irq(vector, cptpf_vf_me_intr, 0, "CPTPF ME1",
cptpf);
if (ret) {
dev_err(dev,
"IRQ registration failed for VFFLR1 irq\n");
goto free_flr1_irq;
}
}
cptpf_enable_vfpf_mbox_intr(cptpf, num_vfs);
cptpf_enable_vf_flr_intrs(cptpf);
cptpf_enable_vf_flr_me_intrs(cptpf, num_vfs);
return 0;
free_flr1_irq:
vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1);
free_irq(vector, cptpf);
free_mbox1_irq:
vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
free_irq(vector, cptpf);
free_me0_irq:
vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME0);
free_irq(vector, cptpf);
free_flr0_irq:
vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0);
free_irq(vector, cptpf);
@ -284,7 +357,11 @@ static int cptpf_vfpf_mbox_init(struct otx2_cptpf_dev *cptpf, int num_vfs)
return -ENOMEM;
/* Map VF-PF mailbox memory */
vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_BAR4_ADDR);
if (test_bit(CN10K_MBOX, &cptpf->cap_flag))
vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_MBOX_ADDR);
else
vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_BAR4_ADDR);
if (!vfpf_mbox_base) {
dev_err(dev, "VF-PF mailbox address not configured\n");
err = -ENOMEM;
@ -365,6 +442,8 @@ static int cptpf_register_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf)
static int cptpf_afpf_mbox_init(struct otx2_cptpf_dev *cptpf)
{
struct pci_dev *pdev = cptpf->pdev;
resource_size_t offset;
int err;
cptpf->afpf_mbox_wq = alloc_workqueue("cpt_afpf_mailbox",
@ -373,8 +452,17 @@ static int cptpf_afpf_mbox_init(struct otx2_cptpf_dev *cptpf)
if (!cptpf->afpf_mbox_wq)
return -ENOMEM;
offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM);
/* Map AF-PF mailbox memory */
cptpf->afpf_mbox_base = devm_ioremap_wc(&pdev->dev, offset, MBOX_SIZE);
if (!cptpf->afpf_mbox_base) {
dev_err(&pdev->dev, "Unable to map BAR4\n");
err = -ENOMEM;
goto error;
}
err = otx2_mbox_init(&cptpf->afpf_mbox, cptpf->afpf_mbox_base,
cptpf->pdev, cptpf->reg_base, MBOX_DIR_PFAF, 1);
pdev, cptpf->reg_base, MBOX_DIR_PFAF, 1);
if (err)
goto error;
@ -570,7 +658,7 @@ static int cptpf_sriov_enable(struct pci_dev *pdev, int num_vfs)
if (ret)
goto disable_intr;
ret = otx2_cpt_create_eng_grps(cptpf->pdev, &cptpf->eng_grps);
ret = otx2_cpt_create_eng_grps(cptpf, &cptpf->eng_grps);
if (ret)
goto disable_intr;
@ -607,7 +695,6 @@ static int otx2_cptpf_probe(struct pci_dev *pdev,
const struct pci_device_id *ent)
{
struct device *dev = &pdev->dev;
resource_size_t offset, size;
struct otx2_cptpf_dev *cptpf;
int err;
@ -644,15 +731,6 @@ static int otx2_cptpf_probe(struct pci_dev *pdev,
if (err)
goto clear_drvdata;
offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM);
size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM);
/* Map AF-PF mailbox memory */
cptpf->afpf_mbox_base = devm_ioremap_wc(dev, offset, size);
if (!cptpf->afpf_mbox_base) {
dev_err(&pdev->dev, "Unable to map BAR4\n");
err = -ENODEV;
goto clear_drvdata;
}
err = pci_alloc_irq_vectors(pdev, RVU_PF_INT_VEC_CNT,
RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX);
if (err < 0) {
@ -660,6 +738,7 @@ static int otx2_cptpf_probe(struct pci_dev *pdev,
RVU_PF_INT_VEC_CNT);
goto clear_drvdata;
}
otx2_cpt_set_hw_caps(pdev, &cptpf->cap_flag);
/* Initialize AF-PF mailbox */
err = cptpf_afpf_mbox_init(cptpf);
if (err)
@ -671,6 +750,10 @@ static int otx2_cptpf_probe(struct pci_dev *pdev,
cptpf->max_vfs = pci_sriov_get_totalvfs(pdev);
err = cn10k_cptpf_lmtst_init(cptpf);
if (err)
goto unregister_intr;
/* Initialize CPT PF device */
err = cptpf_device_init(cptpf);
if (err)
@ -719,6 +802,7 @@ static void otx2_cptpf_remove(struct pci_dev *pdev)
/* Supported devices */
static const struct pci_device_id otx2_cpt_id_table[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OTX2_CPT_PCI_PF_DEVICE_ID) },
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CN10K_CPT_PCI_PF_DEVICE_ID) },
{ 0, } /* end of table */
};

32
drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
View File

@ -16,6 +16,8 @@
#define LOADFVC_MAJOR_OP 0x01
#define LOADFVC_MINOR_OP 0x08
#define CTX_FLUSH_TIMER_CNT 0xFFFFFF
struct fw_info_t {
struct list_head ucodes;
};
@ -666,7 +668,8 @@ static int reserve_engines(struct device *dev,
static void ucode_unload(struct device *dev, struct otx2_cpt_ucode *ucode)
{
if (ucode->va) {
dma_free_coherent(dev, ucode->size, ucode->va, ucode->dma);
dma_free_coherent(dev, OTX2_CPT_UCODE_SZ, ucode->va,
ucode->dma);
ucode->va = NULL;
ucode->dma = 0;
ucode->size = 0;
@ -685,7 +688,7 @@ static int copy_ucode_to_dma_mem(struct device *dev,
u32 i;
/* Allocate DMAable space */
ucode->va = dma_alloc_coherent(dev, ucode->size, &ucode->dma,
ucode->va = dma_alloc_coherent(dev, OTX2_CPT_UCODE_SZ, &ucode->dma,
GFP_KERNEL);
if (!ucode->va)
return -ENOMEM;
@ -1100,11 +1103,12 @@ int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type)
return eng_grp_num;
}
int otx2_cpt_create_eng_grps(struct pci_dev *pdev,
int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
struct otx2_cpt_eng_grps *eng_grps)
{
struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { };
struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} };
struct pci_dev *pdev = cptpf->pdev;
struct fw_info_t fw_info;
int ret;
@ -1180,6 +1184,23 @@ int otx2_cpt_create_eng_grps(struct pci_dev *pdev,
eng_grps->is_grps_created = true;
cpt_ucode_release_fw(&fw_info);
if (is_dev_otx2(pdev))
return 0;
/*
* Configure engine group mask to allow context prefetching
* for the groups.
*/
otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL,
OTX2_CPT_ALL_ENG_GRPS_MASK << 3 | BIT_ULL(16),
BLKADDR_CPT0);
/*
* Set interval to periodically flush dirty data for the next
* CTX cache entry. Set the interval count to maximum supported
* value.
*/
otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTX_FLUSH_TIMER,
CTX_FLUSH_TIMER_CNT, BLKADDR_CPT0);
return 0;
delete_eng_grp:
@ -1460,9 +1481,10 @@ int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf)
iq_cmd.cptr.s.grp = otx2_cpt_get_eng_grp(&cptpf->eng_grps,
etype);
otx2_cpt_fill_inst(&inst, &iq_cmd, rptr_baddr);
otx2_cpt_send_cmd(&inst, 1, &cptpf->lfs.lf[0]);
lfs->ops->send_cmd(&inst, 1, &cptpf->lfs.lf[0]);
while (result->s.compcode == OTX2_CPT_COMPLETION_CODE_INIT)
while (lfs->ops->cpt_get_compcode(result) ==
OTX2_CPT_COMPLETION_CODE_INIT)
cpu_relax();
cptpf->eng_caps[etype].u = be64_to_cpup(rptr);

8
drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h
View File

@ -23,11 +23,13 @@
/* Microcode version string length */
#define OTX2_CPT_UCODE_VER_STR_SZ 44
/* Maximum number of supported engines/cores on OcteonTX2 platform */
#define OTX2_CPT_MAX_ENGINES 128
/* Maximum number of supported engines/cores on OcteonTX2/CN10K platform */
#define OTX2_CPT_MAX_ENGINES 144
#define OTX2_CPT_ENGS_BITMASK_LEN BITS_TO_LONGS(OTX2_CPT_MAX_ENGINES)
#define OTX2_CPT_UCODE_SZ (64 * 1024)
/* Microcode types */
enum otx2_cpt_ucode_type {
OTX2_CPT_AE_UC_TYPE = 1, /* AE-MAIN */
@ -153,7 +155,7 @@ int otx2_cpt_init_eng_grps(struct pci_dev *pdev,
struct otx2_cpt_eng_grps *eng_grps);
void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev,
struct otx2_cpt_eng_grps *eng_grps);
int otx2_cpt_create_eng_grps(struct pci_dev *pdev,
int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
struct otx2_cpt_eng_grps *eng_grps);
int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf);
int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type);

3
drivers/crypto/marvell/octeontx2/otx2_cptvf.h
View File

@ -19,11 +19,14 @@ struct otx2_cptvf_dev {
struct otx2_mbox pfvf_mbox;
struct work_struct pfvf_mbox_work;
struct workqueue_struct *pfvf_mbox_wq;
void *bbuf_base;
unsigned long cap_flag;
};
irqreturn_t otx2_cptvf_pfvf_mbox_intr(int irq, void *arg);
void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work);
int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type);
int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf);
int otx2_cpt_mbox_bbuf_init(struct otx2_cptvf_dev *cptvf, struct pci_dev *pdev);
#endif /* __OTX2_CPTVF_H */

49
drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c
View File

@ -5,9 +5,10 @@
#include "otx2_cptvf.h"
#include "otx2_cptlf.h"
#include "otx2_cptvf_algs.h"
#include "cn10k_cpt.h"
#include <rvu_reg.h>
#define OTX2_CPTVF_DRV_NAME "octeontx2-cptvf"
#define OTX2_CPTVF_DRV_NAME "rvu_cptvf"
static void cptvf_enable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf)
{
@ -70,6 +71,8 @@ static int cptvf_register_interrupts(struct otx2_cptvf_dev *cptvf)
static int cptvf_pfvf_mbox_init(struct otx2_cptvf_dev *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
resource_size_t offset, size;
int ret;
cptvf->pfvf_mbox_wq = alloc_workqueue("cpt_pfvf_mailbox",
@ -78,14 +81,39 @@ static int cptvf_pfvf_mbox_init(struct otx2_cptvf_dev *cptvf)
if (!cptvf->pfvf_mbox_wq)
return -ENOMEM;
if (test_bit(CN10K_MBOX, &cptvf->cap_flag)) {
/* For cn10k platform, VF mailbox region is in its BAR2
* register space
*/
cptvf->pfvf_mbox_base = cptvf->reg_base +
CN10K_CPT_VF_MBOX_REGION;
} else {
offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM);
size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM);
/* Map PF-VF mailbox memory */
cptvf->pfvf_mbox_base = devm_ioremap_wc(&pdev->dev, offset,
size);
if (!cptvf->pfvf_mbox_base) {
dev_err(&pdev->dev, "Unable to map BAR4\n");
ret = -ENOMEM;
goto free_wqe;
}
}
ret = otx2_mbox_init(&cptvf->pfvf_mbox, cptvf->pfvf_mbox_base,
cptvf->pdev, cptvf->reg_base, MBOX_DIR_VFPF, 1);
pdev, cptvf->reg_base, MBOX_DIR_VFPF, 1);
if (ret)
goto free_wqe;
ret = otx2_cpt_mbox_bbuf_init(cptvf, pdev);
if (ret)
goto destroy_mbox;
INIT_WORK(&cptvf->pfvf_mbox_work, otx2_cptvf_pfvf_mbox_handler);
return 0;
destroy_mbox:
otx2_mbox_destroy(&cptvf->pfvf_mbox);
free_wqe:
destroy_workqueue(cptvf->pfvf_mbox_wq);
return ret;
@ -305,7 +333,6 @@ static int otx2_cptvf_probe(struct pci_dev *pdev,
const struct pci_device_id *ent)
{
struct device *dev = &pdev->dev;
resource_size_t offset, size;
struct otx2_cptvf_dev *cptvf;
int ret;
@ -337,15 +364,12 @@ static int otx2_cptvf_probe(struct pci_dev *pdev,
cptvf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM];
offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM);
size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM);
/* Map PF-VF mailbox memory */
cptvf->pfvf_mbox_base = devm_ioremap_wc(dev, offset, size);
if (!cptvf->pfvf_mbox_base) {
dev_err(&pdev->dev, "Unable to map BAR4\n");
ret = -ENODEV;
otx2_cpt_set_hw_caps(pdev, &cptvf->cap_flag);
ret = cn10k_cptvf_lmtst_init(cptvf);
if (ret)
goto clear_drvdata;
}
/* Initialize PF<=>VF mailbox */
ret = cptvf_pfvf_mbox_init(cptvf);
if (ret)
@ -392,6 +416,7 @@ static void otx2_cptvf_remove(struct pci_dev *pdev)
/* Supported devices */
static const struct pci_device_id otx2_cptvf_id_table[] = {
{PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID), 0},
{PCI_VDEVICE(CAVIUM, CN10K_CPT_PCI_VF_DEVICE_ID), 0},
{ 0, } /* end of table */
};
@ -405,6 +430,6 @@ static struct pci_driver otx2_cptvf_pci_driver = {
module_pci_driver(otx2_cptvf_pci_driver);
MODULE_AUTHOR("Marvell");
MODULE_DESCRIPTION("Marvell OcteonTX2 CPT Virtual Function Driver");
MODULE_DESCRIPTION("Marvell RVU CPT Virtual Function Driver");
MODULE_LICENSE("GPL v2");
MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table);

43
drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c
View File

@ -5,6 +5,48 @@
#include "otx2_cptvf.h"
#include <rvu_reg.h>
int otx2_cpt_mbox_bbuf_init(struct otx2_cptvf_dev *cptvf, struct pci_dev *pdev)
{
struct otx2_mbox_dev *mdev;
struct otx2_mbox *otx2_mbox;
cptvf->bbuf_base = devm_kmalloc(&pdev->dev, MBOX_SIZE, GFP_KERNEL);
if (!cptvf->bbuf_base)
return -ENOMEM;
/*
* Overwrite mbox mbase to point to bounce buffer, so that PF/VF
* prepare all mbox messages in bounce buffer instead of directly
* in hw mbox memory.
*/
otx2_mbox = &cptvf->pfvf_mbox;
mdev = &otx2_mbox->dev[0];
mdev->mbase = cptvf->bbuf_base;
return 0;
}
static void otx2_cpt_sync_mbox_bbuf(struct otx2_mbox *mbox, int devid)
{
u16 msgs_offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
void *hw_mbase = mbox->hwbase + (devid * MBOX_SIZE);
struct otx2_mbox_dev *mdev = &mbox->dev[devid];
struct mbox_hdr *hdr;
u64 msg_size;
if (mdev->mbase == hw_mbase)
return;
hdr = hw_mbase + mbox->rx_start;
msg_size = hdr->msg_size;
if (msg_size > mbox->rx_size - msgs_offset)
msg_size = mbox->rx_size - msgs_offset;
/* Copy mbox messages from mbox memory to bounce buffer */
memcpy(mdev->mbase + mbox->rx_start,
hw_mbase + mbox->rx_start, msg_size + msgs_offset);
}
irqreturn_t otx2_cptvf_pfvf_mbox_intr(int __always_unused irq, void *arg)
{
struct otx2_cptvf_dev *cptvf = arg;
@ -106,6 +148,7 @@ void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work)
cptvf = container_of(work, struct otx2_cptvf_dev, pfvf_mbox_work);
pfvf_mbox = &cptvf->pfvf_mbox;
otx2_cpt_sync_mbox_bbuf(pfvf_mbox, 0);
mdev = &pfvf_mbox->dev[0];
rsp_hdr = (struct mbox_hdr *)(mdev->mbase + pfvf_mbox->rx_start);
if (rsp_hdr->num_msgs == 0)

17
drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c
View File

@ -320,7 +320,7 @@ static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
cpt_req->dlen, false);
/* Send CPT command */
otx2_cpt_send_cmd(&cptinst, 1, lf);
lf->lfs->ops->send_cmd(&cptinst, 1, lf);
/*
* We allocate and prepare pending queue entry in critical section
@ -349,13 +349,14 @@ int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
&lfs->lf[cpu_num]);
}
static int cpt_process_ccode(struct pci_dev *pdev,
static int cpt_process_ccode(struct otx2_cptlfs_info *lfs,
union otx2_cpt_res_s *cpt_status,
struct otx2_cpt_inst_info *info,
u32 *res_code)
{
u8 uc_ccode = cpt_status->s.uc_compcode;
u8 ccode = cpt_status->s.compcode;
u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status);
u8 ccode = lfs->ops->cpt_get_compcode(cpt_status);
struct pci_dev *pdev = lfs->pdev;
switch (ccode) {
case OTX2_CPT_COMP_E_FAULT:
@ -389,6 +390,7 @@ static int cpt_process_ccode(struct pci_dev *pdev,
return 1;
case OTX2_CPT_COMP_E_GOOD:
case OTX2_CPT_COMP_E_WARN:
/*
* Check microcode completion code, it is only valid
* when completion code is CPT_COMP_E::GOOD
@ -426,7 +428,7 @@ static int cpt_process_ccode(struct pci_dev *pdev,
return 0;
}
static inline void process_pending_queue(struct pci_dev *pdev,
static inline void process_pending_queue(struct otx2_cptlfs_info *lfs,
struct otx2_cpt_pending_queue *pqueue)
{
struct otx2_cpt_pending_entry *resume_pentry = NULL;
@ -436,6 +438,7 @@ static inline void process_pending_queue(struct pci_dev *pdev,
struct otx2_cpt_inst_info *info = NULL;
struct otx2_cpt_req_info *req = NULL;
struct crypto_async_request *areq;
struct pci_dev *pdev = lfs->pdev;
u32 res_code, resume_index;
while (1) {
@ -476,7 +479,7 @@ static inline void process_pending_queue(struct pci_dev *pdev,
goto process_pentry;
}
if (cpt_process_ccode(pdev, cpt_status, info, &res_code)) {
if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) {
spin_unlock_bh(&pqueue->lock);
return;
}
@ -529,7 +532,7 @@ static inline void process_pending_queue(struct pci_dev *pdev,
void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe)
{
process_pending_queue(wqe->lfs->pdev,
process_pending_queue(wqe->lfs,
&wqe->lfs->lf[wqe->lf_num].pqueue);
}

31
drivers/crypto/nx/nx-842-pseries.c
View File

@ -264,8 +264,8 @@ static int nx842_validate_result(struct device *dev,
* @inlen: Length of input buffer
* @out: Pointer to output buffer
* @outlen: Length of output buffer
* @wrkmem: ptr to buffer for working memory, size determined by
* nx842_pseries_driver.workmem_size
* @wmem: ptr to buffer for working memory, size determined by
* nx842_pseries_driver.workmem_size
*
* Returns:
* 0 Success, output of length @outlen stored in the buffer at @out
@ -393,8 +393,8 @@ static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
* @inlen: Length of input buffer
* @out: Pointer to output buffer
* @outlen: Length of output buffer
* @wrkmem: ptr to buffer for working memory, size determined by
* nx842_pseries_driver.workmem_size
* @wmem: ptr to buffer for working memory, size determined by
* nx842_pseries_driver.workmem_size
*
* Returns:
* 0 Success, output of length @outlen stored in the buffer at @out
@ -513,7 +513,7 @@ static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
/**
* nx842_OF_set_defaults -- Set default (disabled) values for devdata
*
* @devdata - struct nx842_devdata to update
* @devdata: struct nx842_devdata to update
*
* Returns:
* 0 on success
@ -538,13 +538,15 @@ static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
* The status field indicates if the device is enabled when the status
* is 'okay'. Otherwise the device driver will be disabled.
*
* @prop - struct property point containing the maxsyncop for the update
* @devdata: struct nx842_devdata to use for dev_info
* @prop: struct property point containing the maxsyncop for the update
*
* Returns:
* 0 - Device is available
* -ENODEV - Device is not available
*/
static int nx842_OF_upd_status(struct property *prop)
static int nx842_OF_upd_status(struct nx842_devdata *devdata,
struct property *prop)
{
const char *status = (const char *)prop->value;
@ -571,8 +573,8 @@ static int nx842_OF_upd_status(struct property *prop)
* In this example, the maximum byte length of a scatter list is
* 0x0ff0 (4,080).
*
* @devdata - struct nx842_devdata to update
* @prop - struct property point containing the maxsyncop for the update
* @devdata: struct nx842_devdata to update
* @prop: struct property point containing the maxsyncop for the update
*
* Returns:
* 0 on success
@ -619,8 +621,8 @@ static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
* 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
* elements.
*
* @devdata - struct nx842_devdata to update
* @prop - struct property point containing the maxsyncop for the update
* @devdata: struct nx842_devdata to update
* @prop: struct property point containing the maxsyncop for the update
*
* Returns:
* 0 on success
@ -689,7 +691,6 @@ static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
}
/**
*
* nx842_OF_upd -- Handle OF properties updates for the device.
*
* Set all properties from the OF tree. Optionally, a new property
@ -758,7 +759,7 @@ static int nx842_OF_upd(struct property *new_prop)
goto out;
/* Perform property updates */
ret = nx842_OF_upd_status(status);
ret = nx842_OF_upd_status(new_devdata, status);
if (ret)
goto error_out;
@ -812,8 +813,7 @@ static int nx842_OF_upd(struct property *new_prop)
*
* @np: notifier block
* @action: notifier action
* @update: struct pSeries_reconfig_prop_update pointer if action is
* PSERIES_UPDATE_PROPERTY
* @data: struct of_reconfig_data pointer
*
* Returns:
* NOTIFY_OK on success
@ -1069,6 +1069,7 @@ static const struct vio_device_id nx842_vio_driver_ids[] = {
{"ibm,compression-v1", "ibm,compression"},
{"", ""},
};
MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);
static struct vio_driver nx842_vio_driver = {
.name = KBUILD_MODNAME,

2
drivers/crypto/nx/nx-aes-cbc.c
View File

@ -88,7 +88,7 @@ static int cbc_aes_nx_crypt(struct skcipher_request *req,
memcpy(req->iv, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
&(nx_ctx->stats->aes_bytes));
processed += to_process;

4
drivers/crypto/nx/nx-aes-ccm.c
View File

@ -391,7 +391,7 @@ static int ccm_nx_decrypt(struct aead_request *req,
/* update stats */
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
&(nx_ctx->stats->aes_bytes));
processed += to_process;
@ -460,7 +460,7 @@ static int ccm_nx_encrypt(struct aead_request *req,
/* update stats */
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
&(nx_ctx->stats->aes_bytes));
processed += to_process;

4
drivers/crypto/nx/nx-aes-ctr.c
View File

@ -102,7 +102,7 @@ static int ctr_aes_nx_crypt(struct skcipher_request *req, u8 *iv)
memcpy(iv, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
&(nx_ctx->stats->aes_bytes));
processed += to_process;
@ -118,7 +118,7 @@ static int ctr3686_aes_nx_crypt(struct skcipher_request *req)
struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm);
u8 iv[16];
memcpy(iv, nx_ctx->priv.ctr.nonce, CTR_RFC3686_IV_SIZE);
memcpy(iv, nx_ctx->priv.ctr.nonce, CTR_RFC3686_NONCE_SIZE);
memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->iv, CTR_RFC3686_IV_SIZE);
iv[12] = iv[13] = iv[14] = 0;
iv[15] = 1;

2
drivers/crypto/nx/nx-aes-ecb.c
View File

@ -86,7 +86,7 @@ static int ecb_aes_nx_crypt(struct skcipher_request *req,
goto out;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
&(nx_ctx->stats->aes_bytes));
processed += to_process;

2
drivers/crypto/nx/nx-aes-gcm.c
View File

@ -382,7 +382,7 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc,
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
atomic_inc(&(nx_ctx->stats->aes_ops));
atomic64_add(csbcpb->csb.processed_byte_count,
atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
&(nx_ctx->stats->aes_bytes));
processed += to_process;

4
drivers/crypto/nx/nx-common-powernv.c
View File

@ -660,8 +660,8 @@ static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
* @inlen: input buffer size
* @out: output buffer pointer
* @outlenp: output buffer size pointer
* @workmem: working memory buffer pointer, size determined by
* nx842_powernv_driver.workmem_size
* @wmem: working memory buffer pointer, size determined by
* nx842_powernv_driver.workmem_size
*
* Returns: see @nx842_powernv_exec()
*/

19
drivers/crypto/nx/nx-sha256.c
View File

@ -16,6 +16,11 @@
#include "nx_csbcpb.h"
#include "nx.h"
struct sha256_state_be {
__be32 state[SHA256_DIGEST_SIZE / 4];
u64 count;
u8 buf[SHA256_BLOCK_SIZE];
};
static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm)
{
@ -36,7 +41,7 @@ static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm)
}
static int nx_sha256_init(struct shash_desc *desc) {
struct sha256_state *sctx = shash_desc_ctx(desc);
struct sha256_state_be *sctx = shash_desc_ctx(desc);
memset(sctx, 0, sizeof *sctx);
@ -56,7 +61,7 @@ static int nx_sha256_init(struct shash_desc *desc) {
static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
struct sha256_state_be *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *out_sg;
@ -175,7 +180,7 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
static int nx_sha256_final(struct shash_desc *desc, u8 *out)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
struct sha256_state_be *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
@ -245,7 +250,7 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out)
static int nx_sha256_export(struct shash_desc *desc, void *out)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
struct sha256_state_be *sctx = shash_desc_ctx(desc);
memcpy(out, sctx, sizeof(*sctx));
@ -254,7 +259,7 @@ static int nx_sha256_export(struct shash_desc *desc, void *out)
static int nx_sha256_import(struct shash_desc *desc, const void *in)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
struct sha256_state_be *sctx = shash_desc_ctx(desc);
memcpy(sctx, in, sizeof(*sctx));
@ -268,8 +273,8 @@ struct shash_alg nx_shash_sha256_alg = {
.final = nx_sha256_final,
.export = nx_sha256_export,
.import = nx_sha256_import,
.descsize = sizeof(struct sha256_state),
.statesize = sizeof(struct sha256_state),
.descsize = sizeof(struct sha256_state_be),
.statesize = sizeof(struct sha256_state_be),
.base = {
.cra_name = "sha256",
.cra_driver_name = "sha256-nx",

19
drivers/crypto/nx/nx-sha512.c
View File

@ -15,6 +15,11 @@
#include "nx_csbcpb.h"
#include "nx.h"
struct sha512_state_be {
__be64 state[SHA512_DIGEST_SIZE / 8];
u64 count[2];
u8 buf[SHA512_BLOCK_SIZE];
};
static int nx_crypto_ctx_sha512_init(struct crypto_tfm *tfm)
{
@ -36,7 +41,7 @@ static int nx_crypto_ctx_sha512_init(struct crypto_tfm *tfm)
static int nx_sha512_init(struct shash_desc *desc)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct sha512_state_be *sctx = shash_desc_ctx(desc);
memset(sctx, 0, sizeof *sctx);
@ -56,7 +61,7 @@ static int nx_sha512_init(struct shash_desc *desc)
static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct sha512_state_be *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *out_sg;
@ -178,7 +183,7 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
static int nx_sha512_final(struct shash_desc *desc, u8 *out)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct sha512_state_be *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
@ -251,7 +256,7 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out)
static int nx_sha512_export(struct shash_desc *desc, void *out)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct sha512_state_be *sctx = shash_desc_ctx(desc);
memcpy(out, sctx, sizeof(*sctx));
@ -260,7 +265,7 @@ static int nx_sha512_export(struct shash_desc *desc, void *out)
static int nx_sha512_import(struct shash_desc *desc, const void *in)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
struct sha512_state_be *sctx = shash_desc_ctx(desc);
memcpy(sctx, in, sizeof(*sctx));
@ -274,8 +279,8 @@ struct shash_alg nx_shash_sha512_alg = {
.final = nx_sha512_final,
.export = nx_sha512_export,
.import = nx_sha512_import,
.descsize = sizeof(struct sha512_state),
.statesize = sizeof(struct sha512_state),
.descsize = sizeof(struct sha512_state_be),
.statesize = sizeof(struct sha512_state_be),
.base = {
.cra_name = "sha512",
.cra_driver_name = "sha512-nx",

4
drivers/crypto/nx/nx_csbcpb.h
View File

@ -140,8 +140,8 @@ struct cop_status_block {
u8 crb_seq_number;
u8 completion_code;
u8 completion_extension;
u32 processed_byte_count;
u64 address;
__be32 processed_byte_count;
__be64 address;
} __packed;
/* Nest accelerator workbook section 4.4 */

9
drivers/crypto/omap-des.c
View File

@ -229,9 +229,8 @@ static int omap_des_hw_init(struct omap_des_dev *dd)
* It may be long delays between requests.
* Device might go to off mode to save power.
*/
err = pm_runtime_get_sync(dd->dev);
err = pm_runtime_resume_and_get(dd->dev);
if (err < 0) {
pm_runtime_put_noidle(dd->dev);
dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);
return err;
}
@ -994,9 +993,8 @@ static int omap_des_probe(struct platform_device *pdev)
pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
pm_runtime_enable(dev);
err = pm_runtime_get_sync(dev);
err = pm_runtime_resume_and_get(dev);
if (err < 0) {
pm_runtime_put_noidle(dev);
dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);
goto err_get;
}
@ -1124,9 +1122,8 @@ static int omap_des_resume(struct device *dev)
{
int err;
err = pm_runtime_get_sync(dev);
err = pm_runtime_resume_and_get(dev);
if (err < 0) {
pm_runtime_put_noidle(dev);
dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err);
return err;
}

4
drivers/crypto/omap-sham.c
View File

@ -372,7 +372,7 @@ static int omap_sham_hw_init(struct omap_sham_dev *dd)
{
int err;
err = pm_runtime_get_sync(dd->dev);
err = pm_runtime_resume_and_get(dd->dev);
if (err < 0) {
dev_err(dd->dev, "failed to get sync: %d\n", err);
return err;
@ -2244,7 +2244,7 @@ static int omap_sham_suspend(struct device *dev)
static int omap_sham_resume(struct device *dev)
{
int err = pm_runtime_get_sync(dev);
int err = pm_runtime_resume_and_get(dev);
if (err < 0) {
dev_err(dev, "failed to get sync: %d\n", err);
return err;

2
drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h
View File

@ -24,7 +24,7 @@ struct icp_qat_fw_loader_hal_handle {
};
struct icp_qat_fw_loader_chip_info {
bool sram_visible;
int mmp_sram_size;
bool nn;
bool lm2lm3;
u32 lm_size;

14
drivers/crypto/qat/qat_common/qat_hal.c
View File

@ -696,7 +696,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
handle->pci_dev = pci_info->pci_dev;
switch (handle->pci_dev->device) {
case ADF_4XXX_PCI_DEVICE_ID:
handle->chip_info->sram_visible = false;
handle->chip_info->mmp_sram_size = 0;
handle->chip_info->nn = false;
handle->chip_info->lm2lm3 = true;
handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X;
@ -730,7 +730,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
break;
case PCI_DEVICE_ID_INTEL_QAT_C62X:
case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
handle->chip_info->sram_visible = false;
handle->chip_info->mmp_sram_size = 0;
handle->chip_info->nn = true;
handle->chip_info->lm2lm3 = false;
handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
@ -763,7 +763,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
+ LOCAL_TO_XFER_REG_OFFSET);
break;
case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
handle->chip_info->sram_visible = true;
handle->chip_info->mmp_sram_size = 0x40000;
handle->chip_info->nn = true;
handle->chip_info->lm2lm3 = false;
handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
@ -800,7 +800,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
goto out_err;
}
if (handle->chip_info->sram_visible) {
if (handle->chip_info->mmp_sram_size > 0) {
sram_bar =
&pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
handle->hal_sram_addr_v = sram_bar->virt_addr;
@ -1417,7 +1417,11 @@ static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
return -EINVAL;
}
qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
if (status) {
pr_err("QAT: failed to read register");
return status;
}
gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
data16low = 0xffff & data;
data16hi = 0xffff & (data >> 0x10);

12
drivers/crypto/qat/qat_common/qat_uclo.c
View File

@ -342,7 +342,6 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle,
return 0;
}
#define ICP_DH895XCC_PESRAM_BAR_SIZE 0x80000
static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle,
struct icp_qat_uof_initmem *init_mem)
{
@ -1546,15 +1545,14 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle,
int status = 0;
if (handle->chip_info->fw_auth) {
if (!qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc))
status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc);
if (!status)
status = qat_uclo_auth_fw(handle, desc);
qat_uclo_ummap_auth_fw(handle, &desc);
} else {
if (!handle->chip_info->sram_visible) {
dev_dbg(&handle->pci_dev->dev,
"QAT MMP fw not loaded for device 0x%x",
handle->pci_dev->device);
return status;
if (handle->chip_info->mmp_sram_size < mem_size) {
pr_err("QAT: MMP size is too large: 0x%x\n", mem_size);
return -EFBIG;
}
qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size);
}

1
drivers/crypto/qce/Makefile
View File

@ -6,3 +6,4 @@ qcrypto-objs := core.o \
qcrypto-$(CONFIG_CRYPTO_DEV_QCE_SHA) += sha.o
qcrypto-$(CONFIG_CRYPTO_DEV_QCE_SKCIPHER) += skcipher.o
qcrypto-$(CONFIG_CRYPTO_DEV_QCE_AEAD) += aead.o

847
drivers/crypto/qce/aead.c Normal file
View File

@ -0,0 +1,847 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2021, Linaro Limited. All rights reserved.
*/
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <crypto/gcm.h>
#include <crypto/authenc.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/des.h>
#include <crypto/sha1.h>
#include <crypto/sha2.h>
#include <crypto/scatterwalk.h>
#include "aead.h"
#define CCM_NONCE_ADATA_SHIFT 6
#define CCM_NONCE_AUTHSIZE_SHIFT 3
#define MAX_CCM_ADATA_HEADER_LEN 6
static LIST_HEAD(aead_algs);
static void qce_aead_done(void *data)
{
struct crypto_async_request *async_req = data;
struct aead_request *req = aead_request_cast(async_req);
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
struct qce_result_dump *result_buf = qce->dma.result_buf;
enum dma_data_direction dir_src, dir_dst;
bool diff_dst;
int error;
u32 status;
unsigned int totallen;
unsigned char tag[SHA256_DIGEST_SIZE] = {0};
int ret = 0;
diff_dst = (req->src != req->dst) ? true : false;
dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL;
error = qce_dma_terminate_all(&qce->dma);
if (error)
dev_dbg(qce->dev, "aead dma termination error (%d)\n",
error);
if (diff_dst)
dma_unmap_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src);
dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
if (IS_CCM(rctx->flags)) {
if (req->assoclen) {
sg_free_table(&rctx->src_tbl);
if (diff_dst)
sg_free_table(&rctx->dst_tbl);
} else {
if (!(IS_DECRYPT(rctx->flags) && !diff_dst))
sg_free_table(&rctx->dst_tbl);
}
} else {
sg_free_table(&rctx->dst_tbl);
}
error = qce_check_status(qce, &status);
if (error < 0 && (error != -EBADMSG))
dev_err(qce->dev, "aead operation error (%x)\n", status);
if (IS_ENCRYPT(rctx->flags)) {
totallen = req->cryptlen + req->assoclen;
if (IS_CCM(rctx->flags))
scatterwalk_map_and_copy(rctx->ccmresult_buf, req->dst,
totallen, ctx->authsize, 1);
else
scatterwalk_map_and_copy(result_buf->auth_iv, req->dst,
totallen, ctx->authsize, 1);
} else if (!IS_CCM(rctx->flags)) {
totallen = req->cryptlen + req->assoclen - ctx->authsize;
scatterwalk_map_and_copy(tag, req->src, totallen, ctx->authsize, 0);
ret = memcmp(result_buf->auth_iv, tag, ctx->authsize);
if (ret) {
pr_err("Bad message error\n");
error = -EBADMSG;
}
}
qce->async_req_done(qce, error);
}
static struct scatterlist *
qce_aead_prepare_result_buf(struct sg_table *tbl, struct aead_request *req)
{
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ);
return qce_sgtable_add(tbl, &rctx->result_sg, QCE_RESULT_BUF_SZ);
}
static struct scatterlist *
qce_aead_prepare_ccm_result_buf(struct sg_table *tbl, struct aead_request *req)
{
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
sg_init_one(&rctx->result_sg, rctx->ccmresult_buf, QCE_BAM_BURST_SIZE);
return qce_sgtable_add(tbl, &rctx->result_sg, QCE_BAM_BURST_SIZE);
}
static struct scatterlist *
qce_aead_prepare_dst_buf(struct aead_request *req)
{
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
struct scatterlist *sg, *msg_sg, __sg[2];
gfp_t gfp;
unsigned int assoclen = req->assoclen;
unsigned int totallen;
int ret;
totallen = rctx->cryptlen + assoclen;
rctx->dst_nents = sg_nents_for_len(req->dst, totallen);
if (rctx->dst_nents < 0) {
dev_err(qce->dev, "Invalid numbers of dst SG.\n");
return ERR_PTR(-EINVAL);
}
if (IS_CCM(rctx->flags))
rctx->dst_nents += 2;
else
rctx->dst_nents += 1;
gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp);
if (ret)
return ERR_PTR(ret);
if (IS_CCM(rctx->flags) && assoclen) {
/* Get the dst buffer */
msg_sg = scatterwalk_ffwd(__sg, req->dst, assoclen);
sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->adata_sg,
rctx->assoclen);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto dst_tbl_free;
}
/* dst buffer */
sg = qce_sgtable_add(&rctx->dst_tbl, msg_sg, rctx->cryptlen);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto dst_tbl_free;
}
totallen = rctx->cryptlen + rctx->assoclen;
} else {
if (totallen) {
sg = qce_sgtable_add(&rctx->dst_tbl, req->dst, totallen);
if (IS_ERR(sg))
goto dst_tbl_free;
}
}
if (IS_CCM(rctx->flags))
sg = qce_aead_prepare_ccm_result_buf(&rctx->dst_tbl, req);
else
sg = qce_aead_prepare_result_buf(&rctx->dst_tbl, req);
if (IS_ERR(sg))
goto dst_tbl_free;
sg_mark_end(sg);
rctx->dst_sg = rctx->dst_tbl.sgl;
rctx->dst_nents = sg_nents_for_len(rctx->dst_sg, totallen) + 1;
return sg;
dst_tbl_free:
sg_free_table(&rctx->dst_tbl);
return sg;
}
static int
qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req)
{
struct scatterlist *sg, *msg_sg, __sg[2];
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
unsigned int assoclen = rctx->assoclen;
unsigned int adata_header_len, cryptlen, totallen;
gfp_t gfp;
bool diff_dst;
int ret;
if (IS_DECRYPT(rctx->flags))
cryptlen = rctx->cryptlen + ctx->authsize;
else
cryptlen = rctx->cryptlen;
totallen = cryptlen + req->assoclen;
/* Get the msg */
msg_sg = scatterwalk_ffwd(__sg, req->src, req->assoclen);
rctx->adata = kzalloc((ALIGN(assoclen, 16) + MAX_CCM_ADATA_HEADER_LEN) *
sizeof(unsigned char), GFP_ATOMIC);
if (!rctx->adata)
return -ENOMEM;
/*
* Format associated data (RFC3610 and NIST 800-38C)
* Even though specification allows for AAD to be up to 2^64 - 1 bytes,
* the assoclen field in aead_request is unsigned int and thus limits
* the AAD to be up to 2^32 - 1 bytes. So we handle only two scenarios
* while forming the header for AAD.
*/
if (assoclen < 0xff00) {
adata_header_len = 2;
*(__be16 *)rctx->adata = cpu_to_be16(assoclen);
} else {
adata_header_len = 6;
*(__be16 *)rctx->adata = cpu_to_be16(0xfffe);
*(__be32 *)(rctx->adata + 2) = cpu_to_be32(assoclen);
}
/* Copy the associated data */
if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, assoclen),
rctx->adata + adata_header_len,
assoclen) != assoclen)
return -EINVAL;
/* Pad associated data to block size */
rctx->assoclen = ALIGN(assoclen + adata_header_len, 16);
diff_dst = (req->src != req->dst) ? true : false;
if (diff_dst)
rctx->src_nents = sg_nents_for_len(req->src, totallen) + 1;
else
rctx->src_nents = sg_nents_for_len(req->src, totallen) + 2;
gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC;
ret = sg_alloc_table(&rctx->src_tbl, rctx->src_nents, gfp);
if (ret)
return ret;
/* Associated Data */
sg_init_one(&rctx->adata_sg, rctx->adata, rctx->assoclen);
sg = qce_sgtable_add(&rctx->src_tbl, &rctx->adata_sg,
rctx->assoclen);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto err_free;
}
/* src msg */
sg = qce_sgtable_add(&rctx->src_tbl, msg_sg, cryptlen);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto err_free;
}
if (!diff_dst) {
/*
* For decrypt, when src and dst buffers are same, there is already space
* in the buffer for padded 0's which is output in lieu of
* the MAC that is input. So skip the below.
*/
if (!IS_DECRYPT(rctx->flags)) {
sg = qce_aead_prepare_ccm_result_buf(&rctx->src_tbl, req);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto err_free;
}
}
}
sg_mark_end(sg);
rctx->src_sg = rctx->src_tbl.sgl;
totallen = cryptlen + rctx->assoclen;
rctx->src_nents = sg_nents_for_len(rctx->src_sg, totallen);
if (diff_dst) {
sg = qce_aead_prepare_dst_buf(req);
if (IS_ERR(sg)) {
ret = PTR_ERR(sg);
goto err_free;
}
} else {
if (IS_ENCRYPT(rctx->flags))
rctx->dst_nents = rctx->src_nents + 1;
else
rctx->dst_nents = rctx->src_nents;
rctx->dst_sg = rctx->src_sg;
}
return 0;
err_free:
sg_free_table(&rctx->src_tbl);
return ret;
}
static int qce_aead_prepare_buf(struct aead_request *req)
{
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
struct scatterlist *sg;
bool diff_dst = (req->src != req->dst) ? true : false;
unsigned int totallen;
totallen = rctx->cryptlen + rctx->assoclen;
sg = qce_aead_prepare_dst_buf(req);
if (IS_ERR(sg))
return PTR_ERR(sg);
if (diff_dst) {
rctx->src_nents = sg_nents_for_len(req->src, totallen);
if (rctx->src_nents < 0) {
dev_err(qce->dev, "Invalid numbers of src SG.\n");
return -EINVAL;
}
rctx->src_sg = req->src;
} else {
rctx->src_nents = rctx->dst_nents - 1;
rctx->src_sg = rctx->dst_sg;
}
return 0;
}
static int qce_aead_ccm_prepare_buf(struct aead_request *req)
{
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct scatterlist *sg;
bool diff_dst = (req->src != req->dst) ? true : false;
unsigned int cryptlen;
if (rctx->assoclen)
return qce_aead_ccm_prepare_buf_assoclen(req);
if (IS_ENCRYPT(rctx->flags))
return qce_aead_prepare_buf(req);
cryptlen = rctx->cryptlen + ctx->authsize;
if (diff_dst) {
rctx->src_nents = sg_nents_for_len(req->src, cryptlen);
rctx->src_sg = req->src;
sg = qce_aead_prepare_dst_buf(req);
if (IS_ERR(sg))
return PTR_ERR(sg);
} else {
rctx->src_nents = sg_nents_for_len(req->src, cryptlen);
rctx->src_sg = req->src;
rctx->dst_nents = rctx->src_nents;
rctx->dst_sg = rctx->src_sg;
}
return 0;
}
static int qce_aead_create_ccm_nonce(struct qce_aead_reqctx *rctx, struct qce_aead_ctx *ctx)
{
unsigned int msglen_size, ivsize;
u8 msg_len[4];
int i;
if (!rctx || !rctx->iv)
return -EINVAL;
msglen_size = rctx->iv[0] + 1;
/* Verify that msg len size is valid */
if (msglen_size < 2 || msglen_size > 8)
return -EINVAL;
ivsize = rctx->ivsize;
/*
* Clear the msglen bytes in IV.
* Else the h/w engine and nonce will use any stray value pending there.
*/
if (!IS_CCM_RFC4309(rctx->flags)) {
for (i = 0; i < msglen_size; i++)
rctx->iv[ivsize - i - 1] = 0;
}
/*
* The crypto framework encodes cryptlen as unsigned int. Thus, even though
* spec allows for upto 8 bytes to encode msg_len only 4 bytes are needed.
*/
if (msglen_size > 4)
msglen_size = 4;
memcpy(&msg_len[0], &rctx->cryptlen, 4);
memcpy(&rctx->ccm_nonce[0], rctx->iv, rctx->ivsize);
if (rctx->assoclen)
rctx->ccm_nonce[0] |= 1 << CCM_NONCE_ADATA_SHIFT;
rctx->ccm_nonce[0] |= ((ctx->authsize - 2) / 2) <<
CCM_NONCE_AUTHSIZE_SHIFT;
for (i = 0; i < msglen_size; i++)
rctx->ccm_nonce[QCE_MAX_NONCE - i - 1] = msg_len[i];
return 0;
}
static int
qce_aead_async_req_handle(struct crypto_async_request *async_req)
{
struct aead_request *req = aead_request_cast(async_req);
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
enum dma_data_direction dir_src, dir_dst;
bool diff_dst;
int dst_nents, src_nents, ret;
if (IS_CCM_RFC4309(rctx->flags)) {
memset(rctx->ccm_rfc4309_iv, 0, QCE_MAX_IV_SIZE);
rctx->ccm_rfc4309_iv[0] = 3;
memcpy(&rctx->ccm_rfc4309_iv[1], ctx->ccm4309_salt, QCE_CCM4309_SALT_SIZE);
memcpy(&rctx->ccm_rfc4309_iv[4], req->iv, 8);
rctx->iv = rctx->ccm_rfc4309_iv;
rctx->ivsize = AES_BLOCK_SIZE;
} else {
rctx->iv = req->iv;
rctx->ivsize = crypto_aead_ivsize(tfm);
}
if (IS_CCM_RFC4309(rctx->flags))
rctx->assoclen = req->assoclen - 8;
else
rctx->assoclen = req->assoclen;
diff_dst = (req->src != req->dst) ? true : false;
dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL;
if (IS_CCM(rctx->flags)) {
ret = qce_aead_create_ccm_nonce(rctx, ctx);
if (ret)
return ret;
}
if (IS_CCM(rctx->flags))
ret = qce_aead_ccm_prepare_buf(req);
else
ret = qce_aead_prepare_buf(req);
if (ret)
return ret;
dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
if (dst_nents < 0) {
ret = dst_nents;
goto error_free;
}
if (diff_dst) {
src_nents = dma_map_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src);
if (src_nents < 0) {
ret = src_nents;
goto error_unmap_dst;
}
} else {
if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags))
src_nents = dst_nents;
else
src_nents = dst_nents - 1;
}
ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents, rctx->dst_sg, dst_nents,
qce_aead_done, async_req);
if (ret)
goto error_unmap_src;
qce_dma_issue_pending(&qce->dma);
ret = qce_start(async_req, tmpl->crypto_alg_type);
if (ret)
goto error_terminate;
return 0;
error_terminate:
qce_dma_terminate_all(&qce->dma);
error_unmap_src:
if (diff_dst)
dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src);
error_unmap_dst:
dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
error_free:
if (IS_CCM(rctx->flags) && rctx->assoclen) {
sg_free_table(&rctx->src_tbl);
if (diff_dst)
sg_free_table(&rctx->dst_tbl);
} else {
sg_free_table(&rctx->dst_tbl);
}
return ret;
}
static int qce_aead_crypt(struct aead_request *req, int encrypt)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct qce_alg_template *tmpl = to_aead_tmpl(tfm);
unsigned int blocksize = crypto_aead_blocksize(tfm);
rctx->flags = tmpl->alg_flags;
rctx->flags |= encrypt ? QCE_ENCRYPT : QCE_DECRYPT;
if (encrypt)
rctx->cryptlen = req->cryptlen;
else
rctx->cryptlen = req->cryptlen - ctx->authsize;
/* CE does not handle 0 length messages */
if (!rctx->cryptlen) {
if (!(IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)))
ctx->need_fallback = true;
}
/* If fallback is needed, schedule and exit */
if (ctx->need_fallback) {
/* Reset need_fallback in case the same ctx is used for another transaction */
ctx->need_fallback = false;
aead_request_set_tfm(&rctx->fallback_req, ctx->fallback);
aead_request_set_callback(&rctx->fallback_req, req->base.flags,
req->base.complete, req->base.data);
aead_request_set_crypt(&rctx->fallback_req, req->src,
req->dst, req->cryptlen, req->iv);
aead_request_set_ad(&rctx->fallback_req, req->assoclen);
return encrypt ? crypto_aead_encrypt(&rctx->fallback_req) :
crypto_aead_decrypt(&rctx->fallback_req);
}
/*
* CBC algorithms require message lengths to be
* multiples of block size.
*/
if (IS_CBC(rctx->flags) && !IS_ALIGNED(rctx->cryptlen, blocksize))
return -EINVAL;
/* RFC4309 supported AAD size 16 bytes/20 bytes */
if (IS_CCM_RFC4309(rctx->flags))
if (crypto_ipsec_check_assoclen(req->assoclen))
return -EINVAL;
return tmpl->qce->async_req_enqueue(tmpl->qce, &req->base);
}
static int qce_aead_encrypt(struct aead_request *req)
{
return qce_aead_crypt(req, 1);
}
static int qce_aead_decrypt(struct aead_request *req)
{
return qce_aead_crypt(req, 0);
}
static int qce_aead_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
unsigned int keylen)
{
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
unsigned long flags = to_aead_tmpl(tfm)->alg_flags;
if (IS_CCM_RFC4309(flags)) {
if (keylen < QCE_CCM4309_SALT_SIZE)
return -EINVAL;
keylen -= QCE_CCM4309_SALT_SIZE;
memcpy(ctx->ccm4309_salt, key + keylen, QCE_CCM4309_SALT_SIZE);
}
if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_256 && keylen != AES_KEYSIZE_192)
return -EINVAL;
ctx->enc_keylen = keylen;
ctx->auth_keylen = keylen;
memcpy(ctx->enc_key, key, keylen);
memcpy(ctx->auth_key, key, keylen);
if (keylen == AES_KEYSIZE_192)
ctx->need_fallback = true;
return IS_CCM_RFC4309(flags) ?
crypto_aead_setkey(ctx->fallback, key, keylen + QCE_CCM4309_SALT_SIZE) :
crypto_aead_setkey(ctx->fallback, key, keylen);
}
static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen)
{
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct crypto_authenc_keys authenc_keys;
unsigned long flags = to_aead_tmpl(tfm)->alg_flags;
u32 _key[6];
int err;
err = crypto_authenc_extractkeys(&authenc_keys, key, keylen);
if (err)
return err;
if (authenc_keys.enckeylen > QCE_MAX_KEY_SIZE ||
authenc_keys.authkeylen > QCE_MAX_KEY_SIZE)
return -EINVAL;
if (IS_DES(flags)) {
err = verify_aead_des_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen);
if (err)
return err;
} else if (IS_3DES(flags)) {
err = verify_aead_des3_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen);
if (err)
return err;
/*
* The crypto engine does not support any two keys
* being the same for triple des algorithms. The
* verify_skcipher_des3_key does not check for all the
* below conditions. Schedule fallback in this case.
*/
memcpy(_key, authenc_keys.enckey, DES3_EDE_KEY_SIZE);
if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) ||
!((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) ||
!((_key[0] ^ _key[4]) | (_key[1] ^ _key[5])))
ctx->need_fallback = true;
} else if (IS_AES(flags)) {
/* No random key sizes */
if (authenc_keys.enckeylen != AES_KEYSIZE_128 &&
authenc_keys.enckeylen != AES_KEYSIZE_192 &&
authenc_keys.enckeylen != AES_KEYSIZE_256)
return -EINVAL;
if (authenc_keys.enckeylen == AES_KEYSIZE_192)
ctx->need_fallback = true;
}
ctx->enc_keylen = authenc_keys.enckeylen;
ctx->auth_keylen = authenc_keys.authkeylen;
memcpy(ctx->enc_key, authenc_keys.enckey, authenc_keys.enckeylen);
memset(ctx->auth_key, 0, sizeof(ctx->auth_key));
memcpy(ctx->auth_key, authenc_keys.authkey, authenc_keys.authkeylen);
return crypto_aead_setkey(ctx->fallback, key, keylen);
}
static int qce_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
unsigned long flags = to_aead_tmpl(tfm)->alg_flags;
if (IS_CCM(flags)) {
if (authsize < 4 || authsize > 16 || authsize % 2)
return -EINVAL;
if (IS_CCM_RFC4309(flags) && (authsize < 8 || authsize % 4))
return -EINVAL;
}
ctx->authsize = authsize;
return crypto_aead_setauthsize(ctx->fallback, authsize);
}
static int qce_aead_init(struct crypto_aead *tfm)
{
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
ctx->need_fallback = false;
ctx->fallback = crypto_alloc_aead(crypto_tfm_alg_name(&tfm->base),
0, CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(ctx->fallback))
return PTR_ERR(ctx->fallback);
crypto_aead_set_reqsize(tfm, sizeof(struct qce_aead_reqctx) +
crypto_aead_reqsize(ctx->fallback));
return 0;
}
static void qce_aead_exit(struct crypto_aead *tfm)
{
struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
crypto_free_aead(ctx->fallback);
}
struct qce_aead_def {
unsigned long flags;
const char *name;
const char *drv_name;
unsigned int blocksize;
unsigned int chunksize;
unsigned int ivsize;
unsigned int maxauthsize;
};
static const struct qce_aead_def aead_def[] = {
{
.flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC,
.name = "authenc(hmac(sha1),cbc(des))",
.drv_name = "authenc-hmac-sha1-cbc-des-qce",
.blocksize = DES_BLOCK_SIZE,
.ivsize = DES_BLOCK_SIZE,
.maxauthsize = SHA1_DIGEST_SIZE,
},
{
.flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC,
.name = "authenc(hmac(sha1),cbc(des3_ede))",
.drv_name = "authenc-hmac-sha1-cbc-3des-qce",
.blocksize = DES3_EDE_BLOCK_SIZE,
.ivsize = DES3_EDE_BLOCK_SIZE,
.maxauthsize = SHA1_DIGEST_SIZE,
},
{
.flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC,
.name = "authenc(hmac(sha256),cbc(des))",
.drv_name = "authenc-hmac-sha256-cbc-des-qce",
.blocksize = DES_BLOCK_SIZE,
.ivsize = DES_BLOCK_SIZE,
.maxauthsize = SHA256_DIGEST_SIZE,
},
{
.flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC,
.name = "authenc(hmac(sha256),cbc(des3_ede))",
.drv_name = "authenc-hmac-sha256-cbc-3des-qce",
.blocksize = DES3_EDE_BLOCK_SIZE,
.ivsize = DES3_EDE_BLOCK_SIZE,
.maxauthsize = SHA256_DIGEST_SIZE,
},
{
.flags = QCE_ALG_AES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC,
.name = "authenc(hmac(sha256),cbc(aes))",
.drv_name = "authenc-hmac-sha256-cbc-aes-qce",
.blocksize = AES_BLOCK_SIZE,
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = SHA256_DIGEST_SIZE,
},
{
.flags = QCE_ALG_AES | QCE_MODE_CCM,
.name = "ccm(aes)",
.drv_name = "ccm-aes-qce",
.blocksize = 1,
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
},
{
.flags = QCE_ALG_AES | QCE_MODE_CCM | QCE_MODE_CCM_RFC4309,
.name = "rfc4309(ccm(aes))",
.drv_name = "rfc4309-ccm-aes-qce",
.blocksize = 1,
.ivsize = 8,
.maxauthsize = AES_BLOCK_SIZE,
},
};
static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_device *qce)
{
struct qce_alg_template *tmpl;
struct aead_alg *alg;
int ret;
tmpl = kzalloc(sizeof(*tmpl), GFP_KERNEL);
if (!tmpl)
return -ENOMEM;
alg = &tmpl->alg.aead;
snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
def->drv_name);
alg->base.cra_blocksize = def->blocksize;
alg->chunksize = def->chunksize;
alg->ivsize = def->ivsize;
alg->maxauthsize = def->maxauthsize;
if (IS_CCM(def->flags))
alg->setkey = qce_aead_ccm_setkey;
else
alg->setkey = qce_aead_setkey;
alg->setauthsize = qce_aead_setauthsize;
alg->encrypt = qce_aead_encrypt;
alg->decrypt = qce_aead_decrypt;
alg->init = qce_aead_init;
alg->exit = qce_aead_exit;
alg->base.cra_priority = 300;
alg->base.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_ALLOCATES_MEMORY |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
alg->base.cra_ctxsize = sizeof(struct qce_aead_ctx);
alg->base.cra_alignmask = 0;
alg->base.cra_module = THIS_MODULE;
INIT_LIST_HEAD(&tmpl->entry);
tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AEAD;
tmpl->alg_flags = def->flags;
tmpl->qce = qce;
ret = crypto_register_aead(alg);
if (ret) {
kfree(tmpl);
dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name);
return ret;
}
list_add_tail(&tmpl->entry, &aead_algs);
dev_dbg(qce->dev, "%s is registered\n", alg->base.cra_name);
return 0;
}
static void qce_aead_unregister(struct qce_device *qce)
{
struct qce_alg_template *tmpl, *n;
list_for_each_entry_safe(tmpl, n, &aead_algs, entry) {
crypto_unregister_aead(&tmpl->alg.aead);
list_del(&tmpl->entry);
kfree(tmpl);
}
}
static int qce_aead_register(struct qce_device *qce)
{
int ret, i;
for (i = 0; i < ARRAY_SIZE(aead_def); i++) {
ret = qce_aead_register_one(&aead_def[i], qce);
if (ret)
goto err;
}
return 0;
err:
qce_aead_unregister(qce);
return ret;
}
const struct qce_algo_ops aead_ops = {
.type = CRYPTO_ALG_TYPE_AEAD,
.register_algs = qce_aead_register,
.unregister_algs = qce_aead_unregister,
.async_req_handle = qce_aead_async_req_handle,
};

56
drivers/crypto/qce/aead.h Normal file
View File

@ -0,0 +1,56 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2021, Linaro Limited. All rights reserved.
*/
#ifndef _AEAD_H_
#define _AEAD_H_
#include "common.h"
#include "core.h"
#define QCE_MAX_KEY_SIZE 64
#define QCE_CCM4309_SALT_SIZE 3
struct qce_aead_ctx {
u8 enc_key[QCE_MAX_KEY_SIZE];
u8 auth_key[QCE_MAX_KEY_SIZE];
u8 ccm4309_salt[QCE_CCM4309_SALT_SIZE];
unsigned int enc_keylen;
unsigned int auth_keylen;
unsigned int authsize;
bool need_fallback;
struct crypto_aead *fallback;
};
struct qce_aead_reqctx {
unsigned long flags;
u8 *iv;
unsigned int ivsize;
int src_nents;
int dst_nents;
struct scatterlist result_sg;
struct scatterlist adata_sg;
struct sg_table dst_tbl;
struct sg_table src_tbl;
struct scatterlist *dst_sg;
struct scatterlist *src_sg;
unsigned int cryptlen;
unsigned int assoclen;
unsigned char *adata;
u8 ccm_nonce[QCE_MAX_NONCE];
u8 ccmresult_buf[QCE_BAM_BURST_SIZE];
u8 ccm_rfc4309_iv[QCE_MAX_IV_SIZE];
struct aead_request fallback_req;
};
static inline struct qce_alg_template *to_aead_tmpl(struct crypto_aead *tfm)
{
struct aead_alg *alg = crypto_aead_alg(tfm);
return container_of(alg, struct qce_alg_template, alg.aead);
}
extern const struct qce_algo_ops aead_ops;
#endif /* _AEAD_H_ */

196
drivers/crypto/qce/common.c
View File

@ -15,6 +15,7 @@
#include "core.h"
#include "regs-v5.h"
#include "sha.h"
#include "aead.h"
static inline u32 qce_read(struct qce_device *qce, u32 offset)
{
@ -88,17 +89,20 @@ static void qce_setup_config(struct qce_device *qce)
qce_write(qce, REG_CONFIG, config);
}
static inline void qce_crypto_go(struct qce_device *qce)
static inline void qce_crypto_go(struct qce_device *qce, bool result_dump)
{
qce_write(qce, REG_GOPROC, BIT(GO_SHIFT) | BIT(RESULTS_DUMP_SHIFT));
if (result_dump)
qce_write(qce, REG_GOPROC, BIT(GO_SHIFT) | BIT(RESULTS_DUMP_SHIFT));
else
qce_write(qce, REG_GOPROC, BIT(GO_SHIFT));
}
#ifdef CONFIG_CRYPTO_DEV_QCE_SHA
static u32 qce_auth_cfg(unsigned long flags, u32 key_size)
#if defined(CONFIG_CRYPTO_DEV_QCE_SHA) || defined(CONFIG_CRYPTO_DEV_QCE_AEAD)
static u32 qce_auth_cfg(unsigned long flags, u32 key_size, u32 auth_size)
{
u32 cfg = 0;
if (IS_AES(flags) && (IS_CCM(flags) || IS_CMAC(flags)))
if (IS_CCM(flags) || IS_CMAC(flags))
cfg |= AUTH_ALG_AES << AUTH_ALG_SHIFT;
else
cfg |= AUTH_ALG_SHA << AUTH_ALG_SHIFT;
@ -116,15 +120,16 @@ static u32 qce_auth_cfg(unsigned long flags, u32 key_size)
cfg |= AUTH_SIZE_SHA256 << AUTH_SIZE_SHIFT;
else if (IS_CMAC(flags))
cfg |= AUTH_SIZE_ENUM_16_BYTES << AUTH_SIZE_SHIFT;
else if (IS_CCM(flags))
cfg |= (auth_size - 1) << AUTH_SIZE_SHIFT;
if (IS_SHA1(flags) || IS_SHA256(flags))
cfg |= AUTH_MODE_HASH << AUTH_MODE_SHIFT;
else if (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags) ||
IS_CBC(flags) || IS_CTR(flags))
else if (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags))
cfg |= AUTH_MODE_HMAC << AUTH_MODE_SHIFT;
else if (IS_AES(flags) && IS_CCM(flags))
else if (IS_CCM(flags))
cfg |= AUTH_MODE_CCM << AUTH_MODE_SHIFT;
else if (IS_AES(flags) && IS_CMAC(flags))
else if (IS_CMAC(flags))
cfg |= AUTH_MODE_CMAC << AUTH_MODE_SHIFT;
if (IS_SHA(flags) || IS_SHA_HMAC(flags))
@ -133,13 +138,11 @@ static u32 qce_auth_cfg(unsigned long flags, u32 key_size)
if (IS_CCM(flags))
cfg |= QCE_MAX_NONCE_WORDS << AUTH_NONCE_NUM_WORDS_SHIFT;
if (IS_CBC(flags) || IS_CTR(flags) || IS_CCM(flags) ||
IS_CMAC(flags))
cfg |= BIT(AUTH_LAST_SHIFT) | BIT(AUTH_FIRST_SHIFT);
return cfg;
}
#endif
#ifdef CONFIG_CRYPTO_DEV_QCE_SHA
static int qce_setup_regs_ahash(struct crypto_async_request *async_req)
{
struct ahash_request *req = ahash_request_cast(async_req);
@ -168,7 +171,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req)
qce_clear_array(qce, REG_AUTH_KEY0, 16);
qce_clear_array(qce, REG_AUTH_BYTECNT0, 4);
auth_cfg = qce_auth_cfg(rctx->flags, rctx->authklen);
auth_cfg = qce_auth_cfg(rctx->flags, rctx->authklen, digestsize);
}
if (IS_SHA_HMAC(rctx->flags) || IS_CMAC(rctx->flags)) {
@ -196,7 +199,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req)
qce_write_array(qce, REG_AUTH_BYTECNT0,
(u32 *)rctx->byte_count, 2);
auth_cfg = qce_auth_cfg(rctx->flags, 0);
auth_cfg = qce_auth_cfg(rctx->flags, 0, digestsize);
if (rctx->last_blk)
auth_cfg |= BIT(AUTH_LAST_SHIFT);
@ -219,13 +222,13 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req)
config = qce_config_reg(qce, 1);
qce_write(qce, REG_CONFIG, config);
qce_crypto_go(qce);
qce_crypto_go(qce, true);
return 0;
}
#endif
#ifdef CONFIG_CRYPTO_DEV_QCE_SKCIPHER
#if defined(CONFIG_CRYPTO_DEV_QCE_SKCIPHER) || defined(CONFIG_CRYPTO_DEV_QCE_AEAD)
static u32 qce_encr_cfg(unsigned long flags, u32 aes_key_size)
{
u32 cfg = 0;
@ -271,7 +274,9 @@ static u32 qce_encr_cfg(unsigned long flags, u32 aes_key_size)
return cfg;
}
#endif
#ifdef CONFIG_CRYPTO_DEV_QCE_SKCIPHER
static void qce_xts_swapiv(__be32 *dst, const u8 *src, unsigned int ivsize)
{
u8 swap[QCE_AES_IV_LENGTH];
@ -380,7 +385,156 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req)
config = qce_config_reg(qce, 1);
qce_write(qce, REG_CONFIG, config);
qce_crypto_go(qce);
qce_crypto_go(qce, true);
return 0;
}
#endif
#ifdef CONFIG_CRYPTO_DEV_QCE_AEAD
static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = {
SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0
};
static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = {
SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7
};
static unsigned int qce_be32_to_cpu_array(u32 *dst, const u8 *src, unsigned int len)
{
u32 *d = dst;
const u8 *s = src;
unsigned int n;
n = len / sizeof(u32);
for (; n > 0; n--) {
*d = be32_to_cpup((const __be32 *)s);
s += sizeof(u32);
d++;
}
return DIV_ROUND_UP(len, sizeof(u32));
}
static int qce_setup_regs_aead(struct crypto_async_request *async_req)
{
struct aead_request *req = aead_request_cast(async_req);
struct qce_aead_reqctx *rctx = aead_request_ctx(req);
struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm);
struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req));
struct qce_device *qce = tmpl->qce;
u32 enckey[QCE_MAX_CIPHER_KEY_SIZE / sizeof(u32)] = {0};
u32 enciv[QCE_MAX_IV_SIZE / sizeof(u32)] = {0};
u32 authkey[QCE_SHA_HMAC_KEY_SIZE / sizeof(u32)] = {0};
u32 authiv[SHA256_DIGEST_SIZE / sizeof(u32)] = {0};
u32 authnonce[QCE_MAX_NONCE / sizeof(u32)] = {0};
unsigned int enc_keylen = ctx->enc_keylen;
unsigned int auth_keylen = ctx->auth_keylen;
unsigned int enc_ivsize = rctx->ivsize;
unsigned int auth_ivsize = 0;
unsigned int enckey_words, enciv_words;
unsigned int authkey_words, authiv_words, authnonce_words;
unsigned long flags = rctx->flags;
u32 encr_cfg, auth_cfg, config, totallen;
u32 iv_last_word;
qce_setup_config(qce);
/* Write encryption key */
enckey_words = qce_be32_to_cpu_array(enckey, ctx->enc_key, enc_keylen);
qce_write_array(qce, REG_ENCR_KEY0, enckey, enckey_words);
/* Write encryption iv */
enciv_words = qce_be32_to_cpu_array(enciv, rctx->iv, enc_ivsize);
qce_write_array(qce, REG_CNTR0_IV0, enciv, enciv_words);
if (IS_CCM(rctx->flags)) {
iv_last_word = enciv[enciv_words - 1];
qce_write(qce, REG_CNTR3_IV3, iv_last_word + 1);
qce_write_array(qce, REG_ENCR_CCM_INT_CNTR0, (u32 *)enciv, enciv_words);
qce_write(qce, REG_CNTR_MASK, ~0);
qce_write(qce, REG_CNTR_MASK0, ~0);
qce_write(qce, REG_CNTR_MASK1, ~0);
qce_write(qce, REG_CNTR_MASK2, ~0);
}
/* Clear authentication IV and KEY registers of previous values */
qce_clear_array(qce, REG_AUTH_IV0, 16);
qce_clear_array(qce, REG_AUTH_KEY0, 16);
/* Clear byte count */
qce_clear_array(qce, REG_AUTH_BYTECNT0, 4);
/* Write authentication key */
authkey_words = qce_be32_to_cpu_array(authkey, ctx->auth_key, auth_keylen);
qce_write_array(qce, REG_AUTH_KEY0, (u32 *)authkey, authkey_words);
/* Write initial authentication IV only for HMAC algorithms */
if (IS_SHA_HMAC(rctx->flags)) {
/* Write default authentication iv */
if (IS_SHA1_HMAC(rctx->flags)) {
auth_ivsize = SHA1_DIGEST_SIZE;
memcpy(authiv, std_iv_sha1, auth_ivsize);
} else if (IS_SHA256_HMAC(rctx->flags)) {
auth_ivsize = SHA256_DIGEST_SIZE;
memcpy(authiv, std_iv_sha256, auth_ivsize);
}
authiv_words = auth_ivsize / sizeof(u32);
qce_write_array(qce, REG_AUTH_IV0, (u32 *)authiv, authiv_words);
} else if (IS_CCM(rctx->flags)) {
/* Write nonce for CCM algorithms */
authnonce_words = qce_be32_to_cpu_array(authnonce, rctx->ccm_nonce, QCE_MAX_NONCE);
qce_write_array(qce, REG_AUTH_INFO_NONCE0, authnonce, authnonce_words);
}
/* Set up ENCR_SEG_CFG */
encr_cfg = qce_encr_cfg(flags, enc_keylen);
if (IS_ENCRYPT(flags))
encr_cfg |= BIT(ENCODE_SHIFT);
qce_write(qce, REG_ENCR_SEG_CFG, encr_cfg);
/* Set up AUTH_SEG_CFG */
auth_cfg = qce_auth_cfg(rctx->flags, auth_keylen, ctx->authsize);
auth_cfg |= BIT(AUTH_LAST_SHIFT);
auth_cfg |= BIT(AUTH_FIRST_SHIFT);
if (IS_ENCRYPT(flags)) {
if (IS_CCM(rctx->flags))
auth_cfg |= AUTH_POS_BEFORE << AUTH_POS_SHIFT;
else
auth_cfg |= AUTH_POS_AFTER << AUTH_POS_SHIFT;
} else {
if (IS_CCM(rctx->flags))
auth_cfg |= AUTH_POS_AFTER << AUTH_POS_SHIFT;
else
auth_cfg |= AUTH_POS_BEFORE << AUTH_POS_SHIFT;
}
qce_write(qce, REG_AUTH_SEG_CFG, auth_cfg);
totallen = rctx->cryptlen + rctx->assoclen;
/* Set the encryption size and start offset */
if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags))
qce_write(qce, REG_ENCR_SEG_SIZE, rctx->cryptlen + ctx->authsize);
else
qce_write(qce, REG_ENCR_SEG_SIZE, rctx->cryptlen);
qce_write(qce, REG_ENCR_SEG_START, rctx->assoclen & 0xffff);
/* Set the authentication size and start offset */
qce_write(qce, REG_AUTH_SEG_SIZE, totallen);
qce_write(qce, REG_AUTH_SEG_START, 0);
/* Write total length */
if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags))
qce_write(qce, REG_SEG_SIZE, totallen + ctx->authsize);
else
qce_write(qce, REG_SEG_SIZE, totallen);
/* get little endianness */
config = qce_config_reg(qce, 1);
qce_write(qce, REG_CONFIG, config);
/* Start the process */
qce_crypto_go(qce, !IS_CCM(flags));
return 0;
}
@ -396,6 +550,10 @@ int qce_start(struct crypto_async_request *async_req, u32 type)
#ifdef CONFIG_CRYPTO_DEV_QCE_SHA
case CRYPTO_ALG_TYPE_AHASH:
return qce_setup_regs_ahash(async_req);
#endif
#ifdef CONFIG_CRYPTO_DEV_QCE_AEAD
case CRYPTO_ALG_TYPE_AEAD:
return qce_setup_regs_aead(async_req);
#endif
default:
return -EINVAL;
@ -419,6 +577,8 @@ int qce_check_status(struct qce_device *qce, u32 *status)
*/
if (*status & STATUS_ERRORS || !(*status & BIT(OPERATION_DONE_SHIFT)))
ret = -ENXIO;
else if (*status & BIT(MAC_FAILED_SHIFT))
ret = -EBADMSG;
return ret;
}

9
drivers/crypto/qce/common.h
View File

@ -11,6 +11,7 @@
#include <crypto/aes.h>
#include <crypto/hash.h>
#include <crypto/internal/skcipher.h>
#include <crypto/internal/aead.h>
/* xts du size */
#define QCE_SECTOR_SIZE 512
@ -51,9 +52,11 @@
#define QCE_MODE_CCM BIT(12)
#define QCE_MODE_MASK GENMASK(12, 8)
#define QCE_MODE_CCM_RFC4309 BIT(13)
/* cipher encryption/decryption operations */
#define QCE_ENCRYPT BIT(13)
#define QCE_DECRYPT BIT(14)
#define QCE_ENCRYPT BIT(30)
#define QCE_DECRYPT BIT(31)
#define IS_DES(flags) (flags & QCE_ALG_DES)
#define IS_3DES(flags) (flags & QCE_ALG_3DES)
@ -73,6 +76,7 @@
#define IS_CTR(mode) (mode & QCE_MODE_CTR)
#define IS_XTS(mode) (mode & QCE_MODE_XTS)
#define IS_CCM(mode) (mode & QCE_MODE_CCM)
#define IS_CCM_RFC4309(mode) ((mode) & QCE_MODE_CCM_RFC4309)
#define IS_ENCRYPT(dir) (dir & QCE_ENCRYPT)
#define IS_DECRYPT(dir) (dir & QCE_DECRYPT)
@ -85,6 +89,7 @@ struct qce_alg_template {
union {
struct skcipher_alg skcipher;
struct ahash_alg ahash;
struct aead_alg aead;
} alg;
struct qce_device *qce;
const u8 *hash_zero;

4
drivers/crypto/qce/core.c
View File

@ -17,6 +17,7 @@
#include "core.h"
#include "cipher.h"
#include "sha.h"
#include "aead.h"
#define QCE_MAJOR_VERSION5 0x05
#define QCE_QUEUE_LENGTH 1
@ -28,6 +29,9 @@ static const struct qce_algo_ops *qce_ops[] = {
#ifdef CONFIG_CRYPTO_DEV_QCE_SHA
&ahash_ops,
#endif
#ifdef CONFIG_CRYPTO_DEV_QCE_AEAD
&aead_ops,
#endif
};
static void qce_unregister_algs(struct qce_device *qce)

19
drivers/crypto/qce/skcipher.c
View File

@ -72,7 +72,7 @@ qce_skcipher_async_req_handle(struct crypto_async_request *async_req)
struct scatterlist *sg;
bool diff_dst;
gfp_t gfp;
int ret;
int dst_nents, src_nents, ret;
rctx->iv = req->iv;
rctx->ivsize = crypto_skcipher_ivsize(skcipher);
@ -123,21 +123,26 @@ qce_skcipher_async_req_handle(struct crypto_async_request *async_req)
sg_mark_end(sg);
rctx->dst_sg = rctx->dst_tbl.sgl;
ret = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
if (ret < 0)
dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);
if (dst_nents < 0) {
ret = dst_nents;
goto error_free;
}
if (diff_dst) {
ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src);
if (ret < 0)
src_nents = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src);
if (src_nents < 0) {
ret = src_nents;
goto error_unmap_dst;
}
rctx->src_sg = req->src;
} else {
rctx->src_sg = rctx->dst_sg;
src_nents = dst_nents - 1;
}
ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, rctx->src_nents,
rctx->dst_sg, rctx->dst_nents,
ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents,
rctx->dst_sg, dst_nents,
qce_skcipher_done, async_req);
if (ret)
goto error_unmap_src;

50
drivers/crypto/sa2ul.c
View File

@ -1698,7 +1698,6 @@ static void sa_aead_dma_in_callback(void *data)
size_t pl, ml;
int i;
int err = 0;
u16 auth_len;
u32 *mdptr;
sa_sync_from_device(rxd);
@ -1711,13 +1710,10 @@ static void sa_aead_dma_in_callback(void *data)
for (i = 0; i < (authsize / 4); i++)
mdptr[i + 4] = swab32(mdptr[i + 4]);
auth_len = req->assoclen + req->cryptlen;
if (rxd->enc) {
scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1);
} else {
auth_len -= authsize;
start -= authsize;
scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
0);
@ -2300,9 +2296,9 @@ static int sa_dma_init(struct sa_crypto_data *dd)
dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
if (IS_ERR(dd->dma_rx2)) {
dma_release_channel(dd->dma_rx1);
return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
"Unable to request rx2 DMA channel\n");
ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
"Unable to request rx2 DMA channel\n");
goto err_dma_rx2;
}
dd->dma_tx = dma_request_chan(dd->dev, "tx");
@ -2323,28 +2319,31 @@ static int sa_dma_init(struct sa_crypto_data *dd)
if (ret) {
dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
ret);
return ret;
goto err_dma_config;
}
ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
if (ret) {
dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
ret);
return ret;
goto err_dma_config;
}
ret = dmaengine_slave_config(dd->dma_tx, &cfg);
if (ret) {
dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
ret);
return ret;
goto err_dma_config;
}
return 0;
err_dma_config:
dma_release_channel(dd->dma_tx);
err_dma_tx:
dma_release_channel(dd->dma_rx1);
dma_release_channel(dd->dma_rx2);
err_dma_rx2:
dma_release_channel(dd->dma_rx1);
return ret;
}
@ -2385,10 +2384,8 @@ MODULE_DEVICE_TABLE(of, of_match);
static int sa_ul_probe(struct platform_device *pdev)
{
const struct of_device_id *match;
struct device *dev = &pdev->dev;
struct device_node *node = dev->of_node;
struct resource *res;
static void __iomem *saul_base;
struct sa_crypto_data *dev_data;
int ret;
@ -2397,9 +2394,18 @@ static int sa_ul_probe(struct platform_device *pdev)
if (!dev_data)
return -ENOMEM;
dev_data->match_data = of_device_get_match_data(dev);
if (!dev_data->match_data)
return -ENODEV;
saul_base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(saul_base))
return PTR_ERR(saul_base);
sa_k3_dev = dev;
dev_data->dev = dev;
dev_data->pdev = pdev;
dev_data->base = saul_base;
platform_set_drvdata(pdev, dev_data);
dev_set_drvdata(sa_k3_dev, dev_data);
@ -2408,26 +2414,16 @@ static int sa_ul_probe(struct platform_device *pdev)
if (ret < 0) {
dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__,
ret);
pm_runtime_disable(dev);
return ret;
}
sa_init_mem(dev_data);
ret = sa_dma_init(dev_data);
if (ret)
goto disable_pm_runtime;
match = of_match_node(of_match, dev->of_node);
if (!match) {
dev_err(dev, "No compatible match found\n");
return -ENODEV;
}
dev_data->match_data = match->data;
goto destroy_dma_pool;
spin_lock_init(&dev_data->scid_lock);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
saul_base = devm_ioremap_resource(dev, res);
dev_data->base = saul_base;
if (!dev_data->match_data->skip_engine_control) {
u32 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
@ -2454,9 +2450,9 @@ static int sa_ul_probe(struct platform_device *pdev)
dma_release_channel(dev_data->dma_rx1);
dma_release_channel(dev_data->dma_tx);
destroy_dma_pool:
dma_pool_destroy(dev_data->sc_pool);
disable_pm_runtime:
pm_runtime_put_sync(&pdev->dev);
pm_runtime_disable(&pdev->dev);
@ -2467,6 +2463,8 @@ static int sa_ul_remove(struct platform_device *pdev)
{
struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
of_platform_depopulate(&pdev->dev);
sa_unregister_algos(&pdev->dev);
dma_release_channel(dev_data->dma_rx2);

Some files were not shown because too many files have changed in this diff Show More