Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Pull crypto updates from Herbert Xu:
 "Here is the crypto update for 5.3:

  API:
   - Test shash interface directly in testmgr
   - cra_driver_name is now mandatory

  Algorithms:
   - Replace arc4 crypto_cipher with library helper
   - Implement 5 way interleave for ECB, CBC and CTR on arm64
   - Add xxhash
   - Add continuous self-test on noise source to drbg
   - Update jitter RNG

  Drivers:
   - Add support for SHA204A random number generator
   - Add support for 7211 in iproc-rng200
   - Fix fuzz test failures in inside-secure
   - Fix fuzz test failures in talitos
   - Fix fuzz test failures in qat"

* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (143 commits)
  crypto: stm32/hash - remove interruptible condition for dma
  crypto: stm32/hash - Fix hmac issue more than 256 bytes
  crypto: stm32/crc32 - rename driver file
  crypto: amcc - remove memset after dma_alloc_coherent
  crypto: ccp - Switch to SPDX license identifiers
  crypto: ccp - Validate the the error value used to index error messages
  crypto: doc - Fix formatting of new crypto engine content
  crypto: doc - Add parameter documentation
  crypto: arm64/aes-ce - implement 5 way interleave for ECB, CBC and CTR
  crypto: arm64/aes-ce - add 5 way interleave routines
  crypto: talitos - drop icv_ool
  crypto: talitos - fix hash on SEC1.
  crypto: talitos - move struct talitos_edesc into talitos.h
  lib/scatterlist: Fix mapping iterator when sg->offset is greater than PAGE_SIZE
  crypto/NX: Set receive window credits to max number of CRBs in RxFIFO
  crypto: asymmetric_keys - select CRYPTO_HASH where needed
  crypto: serpent - mark __serpent_setkey_sbox noinline
  crypto: testmgr - dynamically allocate crypto_shash
  crypto: testmgr - dynamically allocate testvec_config
  crypto: talitos - eliminate unneeded 'done' functions at build time
  ...
This commit is contained in:
Linus Torvalds 2019-07-08 20:57:08 -07:00
commit 4d2fa8b44b
168 changed files with 4513 additions and 3784 deletions

View File

@ -4,111 +4,89 @@ Code Examples
Code Example For Symmetric Key Cipher Operation
-----------------------------------------------
This code encrypts some data with AES-256-XTS. For sake of example,
all inputs are random bytes, the encryption is done in-place, and it's
assumed the code is running in a context where it can sleep.
::
/* tie all data structures together */
struct skcipher_def {
struct scatterlist sg;
struct crypto_skcipher *tfm;
struct skcipher_request *req;
struct crypto_wait wait;
};
/* Perform cipher operation */
static unsigned int test_skcipher_encdec(struct skcipher_def *sk,
int enc)
{
int rc;
if (enc)
rc = crypto_wait_req(crypto_skcipher_encrypt(sk->req), &sk->wait);
else
rc = crypto_wait_req(crypto_skcipher_decrypt(sk->req), &sk->wait);
if (rc)
pr_info("skcipher encrypt returned with result %d\n", rc);
return rc;
}
/* Initialize and trigger cipher operation */
static int test_skcipher(void)
{
struct skcipher_def sk;
struct crypto_skcipher *skcipher = NULL;
struct crypto_skcipher *tfm = NULL;
struct skcipher_request *req = NULL;
char *scratchpad = NULL;
char *ivdata = NULL;
unsigned char key[32];
int ret = -EFAULT;
u8 *data = NULL;
const size_t datasize = 512; /* data size in bytes */
struct scatterlist sg;
DECLARE_CRYPTO_WAIT(wait);
u8 iv[16]; /* AES-256-XTS takes a 16-byte IV */
u8 key[64]; /* AES-256-XTS takes a 64-byte key */
int err;
skcipher = crypto_alloc_skcipher("cbc-aes-aesni", 0, 0);
if (IS_ERR(skcipher)) {
pr_info("could not allocate skcipher handle\n");
return PTR_ERR(skcipher);
/*
* Allocate a tfm (a transformation object) and set the key.
*
* In real-world use, a tfm and key are typically used for many
* encryption/decryption operations. But in this example, we'll just do a
* single encryption operation with it (which is not very efficient).
*/
tfm = crypto_alloc_skcipher("xts(aes)", 0, 0);
if (IS_ERR(tfm)) {
pr_err("Error allocating xts(aes) handle: %ld\n", PTR_ERR(tfm));
return PTR_ERR(tfm);
}
req = skcipher_request_alloc(skcipher, GFP_KERNEL);
get_random_bytes(key, sizeof(key));
err = crypto_skcipher_setkey(tfm, key, sizeof(key));
if (err) {
pr_err("Error setting key: %d\n", err);
goto out;
}
/* Allocate a request object */
req = skcipher_request_alloc(tfm, GFP_KERNEL);
if (!req) {
pr_info("could not allocate skcipher request\n");
ret = -ENOMEM;
err = -ENOMEM;
goto out;
}
skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
crypto_req_done,
&sk.wait);
/* Prepare the input data */
data = kmalloc(datasize, GFP_KERNEL);
if (!data) {
err = -ENOMEM;
goto out;
}
get_random_bytes(data, datasize);
/* AES 256 with random key */
get_random_bytes(&key, 32);
if (crypto_skcipher_setkey(skcipher, key, 32)) {
pr_info("key could not be set\n");
ret = -EAGAIN;
/* Initialize the IV */
get_random_bytes(iv, sizeof(iv));
/*
* Encrypt the data in-place.
*
* For simplicity, in this example we wait for the request to complete
* before proceeding, even if the underlying implementation is asynchronous.
*
* To decrypt instead of encrypt, just change crypto_skcipher_encrypt() to
* crypto_skcipher_decrypt().
*/
sg_init_one(&sg, data, datasize);
skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
CRYPTO_TFM_REQ_MAY_SLEEP,
crypto_req_done, &wait);
skcipher_request_set_crypt(req, &sg, &sg, datasize, iv);
err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
if (err) {
pr_err("Error encrypting data: %d\n", err);
goto out;
}
/* IV will be random */
ivdata = kmalloc(16, GFP_KERNEL);
if (!ivdata) {
pr_info("could not allocate ivdata\n");
goto out;
}
get_random_bytes(ivdata, 16);
/* Input data will be random */
scratchpad = kmalloc(16, GFP_KERNEL);
if (!scratchpad) {
pr_info("could not allocate scratchpad\n");
goto out;
}
get_random_bytes(scratchpad, 16);
sk.tfm = skcipher;
sk.req = req;
/* We encrypt one block */
sg_init_one(&sk.sg, scratchpad, 16);
skcipher_request_set_crypt(req, &sk.sg, &sk.sg, 16, ivdata);
crypto_init_wait(&sk.wait);
/* encrypt data */
ret = test_skcipher_encdec(&sk, 1);
if (ret)
goto out;
pr_info("Encryption triggered successfully\n");
pr_debug("Encryption was successful\n");
out:
if (skcipher)
crypto_free_skcipher(skcipher);
if (req)
crypto_free_skcipher(tfm);
skcipher_request_free(req);
if (ivdata)
kfree(ivdata);
if (scratchpad)
kfree(scratchpad);
return ret;
kfree(data);
return err;
}

View File

@ -5,7 +5,7 @@ Block Cipher Algorithm Definitions
:doc: Block Cipher Algorithm Definitions
.. kernel-doc:: include/linux/crypto.h
:functions: crypto_alg ablkcipher_alg blkcipher_alg cipher_alg
:functions: crypto_alg ablkcipher_alg blkcipher_alg cipher_alg compress_alg
Symmetric Key Cipher API
------------------------

View File

@ -208,9 +208,7 @@ the aforementioned cipher types:
- CRYPTO_ALG_TYPE_KPP Key-agreement Protocol Primitive (KPP) such as
an ECDH or DH implementation
- CRYPTO_ALG_TYPE_DIGEST Raw message digest
- CRYPTO_ALG_TYPE_HASH Alias for CRYPTO_ALG_TYPE_DIGEST
- CRYPTO_ALG_TYPE_HASH Raw message digest
- CRYPTO_ALG_TYPE_SHASH Synchronous multi-block hash

View File

@ -1,50 +1,85 @@
=============
CRYPTO ENGINE
.. SPDX-License-Identifier: GPL-2.0
Crypto Engine
=============
Overview
--------
The crypto engine API (CE), is a crypto queue manager.
The crypto engine (CE) API is a crypto queue manager.
Requirement
-----------
You have to put at start of your tfm_ctx the struct crypto_engine_ctx::
You must put, at the start of your transform context your_tfm_ctx, the structure
crypto_engine:
::
struct your_tfm_ctx {
struct crypto_engine_ctx enginectx;
struct crypto_engine engine;
...
};
Why: Since CE manage only crypto_async_request, it cannot know the underlying
request_type and so have access only on the TFM.
So using container_of for accessing __ctx is impossible.
Furthermore, the crypto engine cannot know the "struct your_tfm_ctx",
so it must assume that crypto_engine_ctx is at start of it.
The crypto engine only manages asynchronous requests in the form of
crypto_async_request. It cannot know the underlying request type and thus only
has access to the transform structure. It is not possible to access the context
using container_of. In addition, the engine knows nothing about your
structure "``struct your_tfm_ctx``". The engine assumes (requires) the placement
of the known member ``struct crypto_engine`` at the beginning.
Order of operations
-------------------
You have to obtain a struct crypto_engine via crypto_engine_alloc_init().
And start it via crypto_engine_start().
You are required to obtain a struct crypto_engine via ``crypto_engine_alloc_init()``.
Start it via ``crypto_engine_start()``. When finished with your work, shut down the
engine using ``crypto_engine_stop()`` and destroy the engine with
``crypto_engine_exit()``.
Before transferring any request, you have to fill the enginectx.
- prepare_request: (taking a function pointer) If you need to do some processing before doing the request
- unprepare_request: (taking a function pointer) Undoing what's done in prepare_request
- do_one_request: (taking a function pointer) Do encryption for current request
Before transferring any request, you have to fill the context enginectx by
providing functions for the following:
Note: that those three functions get the crypto_async_request associated with the received request.
So your need to get the original request via container_of(areq, struct yourrequesttype_request, base);
* ``prepare_crypt_hardware``: Called once before any prepare functions are
called.
When your driver receive a crypto_request, you have to transfer it to
* ``unprepare_crypt_hardware``: Called once after all unprepare functions have
been called.
* ``prepare_cipher_request``/``prepare_hash_request``: Called before each
corresponding request is performed. If some processing or other preparatory
work is required, do it here.
* ``unprepare_cipher_request``/``unprepare_hash_request``: Called after each
request is handled. Clean up / undo what was done in the prepare function.
* ``cipher_one_request``/``hash_one_request``: Handle the current request by
performing the operation.
Note that these functions access the crypto_async_request structure
associated with the received request. You are able to retrieve the original
request by using:
::
container_of(areq, struct yourrequesttype_request, base);
When your driver receives a crypto_request, you must to transfer it to
the crypto engine via one of:
- crypto_transfer_ablkcipher_request_to_engine()
- crypto_transfer_aead_request_to_engine()
- crypto_transfer_akcipher_request_to_engine()
- crypto_transfer_hash_request_to_engine()
- crypto_transfer_skcipher_request_to_engine()
At the end of the request process, a call to one of the following function is needed:
- crypto_finalize_ablkcipher_request
- crypto_finalize_aead_request
- crypto_finalize_akcipher_request
- crypto_finalize_hash_request
- crypto_finalize_skcipher_request
* crypto_transfer_ablkcipher_request_to_engine()
* crypto_transfer_aead_request_to_engine()
* crypto_transfer_akcipher_request_to_engine()
* crypto_transfer_hash_request_to_engine()
* crypto_transfer_skcipher_request_to_engine()
At the end of the request process, a call to one of the following functions is needed:
* crypto_finalize_ablkcipher_request()
* crypto_finalize_aead_request()
* crypto_finalize_akcipher_request()
* crypto_finalize_hash_request()
* crypto_finalize_skcipher_request()

View File

@ -66,16 +66,3 @@ sha@f8034000 {
dmas = <&dma1 2 17>;
dma-names = "tx";
};
* Eliptic Curve Cryptography (I2C)
Required properties:
- compatible : must be "atmel,atecc508a".
- reg: I2C bus address of the device.
- clock-frequency: must be present in the i2c controller node.
Example:
atecc508a@c0 {
compatible = "atmel,atecc508a";
reg = <0xC0>;
};

View File

@ -2,6 +2,7 @@ HWRNG support for the iproc-rng200 driver
Required properties:
- compatible : Must be one of:
"brcm,bcm7211-rng200"
"brcm,bcm7278-rng200"
"brcm,iproc-rng200"
- reg : base address and size of control register block

View File

@ -52,6 +52,10 @@ properties:
- at,24c08
# i2c trusted platform module (TPM)
- atmel,at97sc3204t
# i2c h/w symmetric crypto module
- atmel,atsha204a
# i2c h/w elliptic curve crypto module
- atmel,atecc508a
# CM32181: Ambient Light Sensor
- capella,cm32181
# CM3232: Ambient Light Sensor

View File

@ -4257,6 +4257,7 @@ F: crypto/
F: drivers/crypto/
F: include/crypto/
F: include/linux/crypto*
F: lib/crypto/
CRYPTOGRAPHIC RANDOM NUMBER GENERATOR
M: Neil Horman <nhorman@tuxdriver.com>

View File

@ -100,6 +100,29 @@
reg = <0x40000000 0x800000>;
ranges;
crypto: crypto@40240000 {
compatible = "fsl,sec-v4.0";
#address-cells = <1>;
#size-cells = <1>;
reg = <0x40240000 0x10000>;
ranges = <0 0x40240000 0x10000>;
clocks = <&pcc2 IMX7ULP_CLK_CAAM>,
<&scg1 IMX7ULP_CLK_NIC1_BUS_DIV>;
clock-names = "aclk", "ipg";
sec_jr0: jr0@1000 {
compatible = "fsl,sec-v4.0-job-ring";
reg = <0x1000 0x1000>;
interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
};
sec_jr1: jr1@2000 {
compatible = "fsl,sec-v4.0-job-ring";
reg = <0x2000 0x1000>;
interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
};
};
lpuart4: serial@402d0000 {
compatible = "fsl,imx7ulp-lpuart";
reg = <0x402d0000 0x1000>;

View File

@ -63,7 +63,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
}
static int chacha_neon_stream_xor(struct skcipher_request *req,
struct chacha_ctx *ctx, u8 *iv)
const struct chacha_ctx *ctx, const u8 *iv)
{
struct skcipher_walk walk;
u32 state[16];

View File

@ -34,7 +34,7 @@ int sha512_arm_update(struct shash_desc *desc, const u8 *data,
(sha512_block_fn *)sha512_block_data_order);
}
int sha512_arm_final(struct shash_desc *desc, u8 *out)
static int sha512_arm_final(struct shash_desc *desc, u8 *out)
{
sha512_base_do_finalize(desc,
(sha512_block_fn *)sha512_block_data_order);

View File

@ -15,6 +15,8 @@
.arch armv8-a+crypto
xtsmask .req v16
cbciv .req v16
vctr .req v16
.macro xts_reload_mask, tmp
.endm
@ -49,7 +51,7 @@
load_round_keys \rounds, \temp
.endm
.macro do_enc_Nx, de, mc, k, i0, i1, i2, i3
.macro do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4
aes\de \i0\().16b, \k\().16b
aes\mc \i0\().16b, \i0\().16b
.ifnb \i1
@ -60,27 +62,34 @@
aes\mc \i2\().16b, \i2\().16b
aes\de \i3\().16b, \k\().16b
aes\mc \i3\().16b, \i3\().16b
.ifnb \i4
aes\de \i4\().16b, \k\().16b
aes\mc \i4\().16b, \i4\().16b
.endif
.endif
.endif
.endm
/* up to 4 interleaved encryption rounds with the same round key */
.macro round_Nx, enc, k, i0, i1, i2, i3
/* up to 5 interleaved encryption rounds with the same round key */
.macro round_Nx, enc, k, i0, i1, i2, i3, i4
.ifc \enc, e
do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3
do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3, \i4
.else
do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3
do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3, \i4
.endif
.endm
/* up to 4 interleaved final rounds */
.macro fin_round_Nx, de, k, k2, i0, i1, i2, i3
/* up to 5 interleaved final rounds */
.macro fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4
aes\de \i0\().16b, \k\().16b
.ifnb \i1
aes\de \i1\().16b, \k\().16b
.ifnb \i3
aes\de \i2\().16b, \k\().16b
aes\de \i3\().16b, \k\().16b
.ifnb \i4
aes\de \i4\().16b, \k\().16b
.endif
.endif
.endif
eor \i0\().16b, \i0\().16b, \k2\().16b
@ -89,47 +98,52 @@
.ifnb \i3
eor \i2\().16b, \i2\().16b, \k2\().16b
eor \i3\().16b, \i3\().16b, \k2\().16b
.ifnb \i4
eor \i4\().16b, \i4\().16b, \k2\().16b
.endif
.endif
.endif
.endm
/* up to 4 interleaved blocks */
.macro do_block_Nx, enc, rounds, i0, i1, i2, i3
/* up to 5 interleaved blocks */
.macro do_block_Nx, enc, rounds, i0, i1, i2, i3, i4
cmp \rounds, #12
blo 2222f /* 128 bits */
beq 1111f /* 192 bits */
round_Nx \enc, v17, \i0, \i1, \i2, \i3
round_Nx \enc, v18, \i0, \i1, \i2, \i3
1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3
round_Nx \enc, v20, \i0, \i1, \i2, \i3
round_Nx \enc, v17, \i0, \i1, \i2, \i3, \i4
round_Nx \enc, v18, \i0, \i1, \i2, \i3, \i4
1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3, \i4
round_Nx \enc, v20, \i0, \i1, \i2, \i3, \i4
2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29
round_Nx \enc, \key, \i0, \i1, \i2, \i3
round_Nx \enc, \key, \i0, \i1, \i2, \i3, \i4
.endr
fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3
fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3, \i4
.endm
.macro encrypt_block, in, rounds, t0, t1, t2
do_block_Nx e, \rounds, \in
.endm
.macro encrypt_block2x, i0, i1, rounds, t0, t1, t2
do_block_Nx e, \rounds, \i0, \i1
.endm
.macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
do_block_Nx e, \rounds, \i0, \i1, \i2, \i3
.endm
.macro encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
do_block_Nx e, \rounds, \i0, \i1, \i2, \i3, \i4
.endm
.macro decrypt_block, in, rounds, t0, t1, t2
do_block_Nx d, \rounds, \in
.endm
.macro decrypt_block2x, i0, i1, rounds, t0, t1, t2
do_block_Nx d, \rounds, \i0, \i1
.endm
.macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
do_block_Nx d, \rounds, \i0, \i1, \i2, \i3
.endm
.macro decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
do_block_Nx d, \rounds, \i0, \i1, \i2, \i3, \i4
.endm
#define MAX_STRIDE 5
#include "aes-modes.S"

View File

@ -10,6 +10,18 @@
.text
.align 4
#ifndef MAX_STRIDE
#define MAX_STRIDE 4
#endif
#if MAX_STRIDE == 4
#define ST4(x...) x
#define ST5(x...)
#else
#define ST4(x...)
#define ST5(x...) x
#endif
aes_encrypt_block4x:
encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
ret
@ -20,6 +32,18 @@ aes_decrypt_block4x:
ret
ENDPROC(aes_decrypt_block4x)
#if MAX_STRIDE == 5
aes_encrypt_block5x:
encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
ret
ENDPROC(aes_encrypt_block5x)
aes_decrypt_block5x:
decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
ret
ENDPROC(aes_decrypt_block5x)
#endif
/*
* aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks)
@ -34,14 +58,17 @@ AES_ENTRY(aes_ecb_encrypt)
enc_prepare w3, x2, x5
.LecbencloopNx:
subs w4, w4, #4
subs w4, w4, #MAX_STRIDE
bmi .Lecbenc1x
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
bl aes_encrypt_block4x
ST4( bl aes_encrypt_block4x )
ST5( ld1 {v4.16b}, [x1], #16 )
ST5( bl aes_encrypt_block5x )
st1 {v0.16b-v3.16b}, [x0], #64
ST5( st1 {v4.16b}, [x0], #16 )
b .LecbencloopNx
.Lecbenc1x:
adds w4, w4, #4
adds w4, w4, #MAX_STRIDE
beq .Lecbencout
.Lecbencloop:
ld1 {v0.16b}, [x1], #16 /* get next pt block */
@ -62,14 +89,17 @@ AES_ENTRY(aes_ecb_decrypt)
dec_prepare w3, x2, x5
.LecbdecloopNx:
subs w4, w4, #4
subs w4, w4, #MAX_STRIDE
bmi .Lecbdec1x
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
bl aes_decrypt_block4x
ST4( bl aes_decrypt_block4x )
ST5( ld1 {v4.16b}, [x1], #16 )
ST5( bl aes_decrypt_block5x )
st1 {v0.16b-v3.16b}, [x0], #64
ST5( st1 {v4.16b}, [x0], #16 )
b .LecbdecloopNx
.Lecbdec1x:
adds w4, w4, #4
adds w4, w4, #MAX_STRIDE
beq .Lecbdecout
.Lecbdecloop:
ld1 {v0.16b}, [x1], #16 /* get next ct block */
@ -129,39 +159,56 @@ AES_ENTRY(aes_cbc_decrypt)
stp x29, x30, [sp, #-16]!
mov x29, sp
ld1 {v7.16b}, [x5] /* get iv */
ld1 {cbciv.16b}, [x5] /* get iv */
dec_prepare w3, x2, x6
.LcbcdecloopNx:
subs w4, w4, #4
subs w4, w4, #MAX_STRIDE
bmi .Lcbcdec1x
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
#if MAX_STRIDE == 5
ld1 {v4.16b}, [x1], #16 /* get 1 ct block */
mov v5.16b, v0.16b
mov v6.16b, v1.16b
mov v7.16b, v2.16b
bl aes_decrypt_block5x
sub x1, x1, #32
eor v0.16b, v0.16b, cbciv.16b
eor v1.16b, v1.16b, v5.16b
ld1 {v5.16b}, [x1], #16 /* reload 1 ct block */
ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */
eor v2.16b, v2.16b, v6.16b
eor v3.16b, v3.16b, v7.16b
eor v4.16b, v4.16b, v5.16b
#else
mov v4.16b, v0.16b
mov v5.16b, v1.16b
mov v6.16b, v2.16b
bl aes_decrypt_block4x
sub x1, x1, #16
eor v0.16b, v0.16b, v7.16b
eor v0.16b, v0.16b, cbciv.16b
eor v1.16b, v1.16b, v4.16b
ld1 {v7.16b}, [x1], #16 /* reload 1 ct block */
ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */
eor v2.16b, v2.16b, v5.16b
eor v3.16b, v3.16b, v6.16b
#endif
st1 {v0.16b-v3.16b}, [x0], #64
ST5( st1 {v4.16b}, [x0], #16 )
b .LcbcdecloopNx
.Lcbcdec1x:
adds w4, w4, #4
adds w4, w4, #MAX_STRIDE
beq .Lcbcdecout
.Lcbcdecloop:
ld1 {v1.16b}, [x1], #16 /* get next ct block */
mov v0.16b, v1.16b /* ...and copy to v0 */
decrypt_block v0, w3, x2, x6, w7
eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */
mov v7.16b, v1.16b /* ct is next iv */
eor v0.16b, v0.16b, cbciv.16b /* xor with iv => pt */
mov cbciv.16b, v1.16b /* ct is next iv */
st1 {v0.16b}, [x0], #16
subs w4, w4, #1
bne .Lcbcdecloop
.Lcbcdecout:
st1 {v7.16b}, [x5] /* return iv */
st1 {cbciv.16b}, [x5] /* return iv */
ldp x29, x30, [sp], #16
ret
AES_ENDPROC(aes_cbc_decrypt)
@ -255,51 +302,60 @@ AES_ENTRY(aes_ctr_encrypt)
mov x29, sp
enc_prepare w3, x2, x6
ld1 {v4.16b}, [x5]
ld1 {vctr.16b}, [x5]
umov x6, v4.d[1] /* keep swabbed ctr in reg */
umov x6, vctr.d[1] /* keep swabbed ctr in reg */
rev x6, x6
cmn w6, w4 /* 32 bit overflow? */
bcs .Lctrloop
.LctrloopNx:
subs w4, w4, #4
subs w4, w4, #MAX_STRIDE
bmi .Lctr1x
add w7, w6, #1
mov v0.16b, v4.16b
mov v0.16b, vctr.16b
add w8, w6, #2
mov v1.16b, v4.16b
mov v1.16b, vctr.16b
add w9, w6, #3
mov v2.16b, vctr.16b
add w9, w6, #3
mov v2.16b, v4.16b
rev w7, w7
mov v3.16b, v4.16b
mov v3.16b, vctr.16b
rev w8, w8
ST5( mov v4.16b, vctr.16b )
mov v1.s[3], w7
rev w9, w9
ST5( add w10, w6, #4 )
mov v2.s[3], w8
ST5( rev w10, w10 )
mov v3.s[3], w9
ST5( mov v4.s[3], w10 )
ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */
bl aes_encrypt_block4x
ST4( bl aes_encrypt_block4x )
ST5( bl aes_encrypt_block5x )
eor v0.16b, v5.16b, v0.16b
ld1 {v5.16b}, [x1], #16 /* get 1 input block */
ST4( ld1 {v5.16b}, [x1], #16 )
eor v1.16b, v6.16b, v1.16b
ST5( ld1 {v5.16b-v6.16b}, [x1], #32 )
eor v2.16b, v7.16b, v2.16b
eor v3.16b, v5.16b, v3.16b
ST5( eor v4.16b, v6.16b, v4.16b )
st1 {v0.16b-v3.16b}, [x0], #64
add x6, x6, #4
ST5( st1 {v4.16b}, [x0], #16 )
add x6, x6, #MAX_STRIDE
rev x7, x6
ins v4.d[1], x7
ins vctr.d[1], x7
cbz w4, .Lctrout
b .LctrloopNx
.Lctr1x:
adds w4, w4, #4
adds w4, w4, #MAX_STRIDE
beq .Lctrout
.Lctrloop:
mov v0.16b, v4.16b
mov v0.16b, vctr.16b
encrypt_block v0, w3, x2, x8, w7
adds x6, x6, #1 /* increment BE ctr */
rev x7, x6
ins v4.d[1], x7
ins vctr.d[1], x7
bcs .Lctrcarry /* overflow? */
.Lctrcarrydone:
@ -311,7 +367,7 @@ AES_ENTRY(aes_ctr_encrypt)
bne .Lctrloop
.Lctrout:
st1 {v4.16b}, [x5] /* return next CTR value */
st1 {vctr.16b}, [x5] /* return next CTR value */
ldp x29, x30, [sp], #16
ret
@ -320,11 +376,11 @@ AES_ENTRY(aes_ctr_encrypt)
b .Lctrout
.Lctrcarry:
umov x7, v4.d[0] /* load upper word of ctr */
umov x7, vctr.d[0] /* load upper word of ctr */
rev x7, x7 /* ... to handle the carry */
add x7, x7, #1
rev x7, x7
ins v4.d[0], x7
ins vctr.d[0], x7
b .Lctrcarrydone
AES_ENDPROC(aes_ctr_encrypt)

View File

@ -12,6 +12,8 @@
#define AES_ENDPROC(func) ENDPROC(neon_ ## func)
xtsmask .req v7
cbciv .req v7
vctr .req v4
.macro xts_reload_mask, tmp
xts_load_mask \tmp
@ -114,26 +116,9 @@
/*
* Interleaved versions: functionally equivalent to the
* ones above, but applied to 2 or 4 AES states in parallel.
* ones above, but applied to AES states in parallel.
*/
.macro sub_bytes_2x, in0, in1
sub v8.16b, \in0\().16b, v15.16b
tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
sub v9.16b, \in1\().16b, v15.16b
tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b
sub v10.16b, v8.16b, v15.16b
tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b
sub v11.16b, v9.16b, v15.16b
tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b
sub v8.16b, v10.16b, v15.16b
tbx \in0\().16b, {v24.16b-v27.16b}, v10.16b
sub v9.16b, v11.16b, v15.16b
tbx \in1\().16b, {v24.16b-v27.16b}, v11.16b
tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b
tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b
.endm
.macro sub_bytes_4x, in0, in1, in2, in3
sub v8.16b, \in0\().16b, v15.16b
tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
@ -212,25 +197,6 @@
eor \in1\().16b, \in1\().16b, v11.16b
.endm
.macro do_block_2x, enc, in0, in1, rounds, rk, rkp, i
ld1 {v15.4s}, [\rk]
add \rkp, \rk, #16
mov \i, \rounds
1111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
movi v15.16b, #0x40
tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */
tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */
sub_bytes_2x \in0, \in1
subs \i, \i, #1
ld1 {v15.4s}, [\rkp], #16
beq 2222f
mix_columns_2x \in0, \in1, \enc
b 1111b
2222: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
.endm
.macro do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
ld1 {v15.4s}, [\rk]
add \rkp, \rk, #16
@ -257,14 +223,6 @@
eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */
.endm
.macro encrypt_block2x, in0, in1, rounds, rk, rkp, i
do_block_2x 1, \in0, \in1, \rounds, \rk, \rkp, \i
.endm
.macro decrypt_block2x, in0, in1, rounds, rk, rkp, i
do_block_2x 0, \in0, \in1, \rounds, \rk, \rkp, \i
.endm
.macro encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
do_block_4x 1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
.endm

View File

@ -60,7 +60,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
}
static int chacha_neon_stream_xor(struct skcipher_request *req,
struct chacha_ctx *ctx, u8 *iv)
const struct chacha_ctx *ctx, const u8 *iv)
{
struct skcipher_walk walk;
u32 state[16];

View File

@ -52,7 +52,7 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *out)
{
struct sha1_ce_state *sctx = shash_desc_ctx(desc);
bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE);
bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE) && len;
if (!crypto_simd_usable())
return crypto_sha1_finup(desc, data, len, out);

View File

@ -57,7 +57,7 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *out)
{
struct sha256_ce_state *sctx = shash_desc_ctx(desc);
bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE);
bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE) && len;
if (!crypto_simd_usable()) {
if (len)

View File

@ -371,20 +371,6 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
}
}
static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
aesni_enc(ctx, dst, src);
}
static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
aesni_dec(ctx, dst, src);
}
static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int len)
{
@ -920,7 +906,7 @@ static int helper_rfc4106_decrypt(struct aead_request *req)
}
#endif
static struct crypto_alg aesni_algs[] = { {
static struct crypto_alg aesni_cipher_alg = {
.cra_name = "aes",
.cra_driver_name = "aes-aesni",
.cra_priority = 300,
@ -937,24 +923,7 @@ static struct crypto_alg aesni_algs[] = { {
.cia_decrypt = aes_decrypt
}
}
}, {
.cra_name = "__aes",
.cra_driver_name = "__aes-aesni",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = CRYPTO_AES_CTX_SIZE,
.cra_module = THIS_MODULE,
.cra_u = {
.cipher = {
.cia_min_keysize = AES_MIN_KEY_SIZE,
.cia_max_keysize = AES_MAX_KEY_SIZE,
.cia_setkey = aes_set_key,
.cia_encrypt = __aes_encrypt,
.cia_decrypt = __aes_decrypt
}
}
} };
};
static struct skcipher_alg aesni_skciphers[] = {
{
@ -1150,7 +1119,7 @@ static int __init aesni_init(void)
#endif
#endif
err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
err = crypto_register_alg(&aesni_cipher_alg);
if (err)
return err;
@ -1158,7 +1127,7 @@ static int __init aesni_init(void)
ARRAY_SIZE(aesni_skciphers),
aesni_simd_skciphers);
if (err)
goto unregister_algs;
goto unregister_cipher;
err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
aesni_simd_aeads);
@ -1170,8 +1139,8 @@ static int __init aesni_init(void)
unregister_skciphers:
simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
aesni_simd_skciphers);
unregister_algs:
crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
unregister_cipher:
crypto_unregister_alg(&aesni_cipher_alg);
return err;
}
@ -1181,7 +1150,7 @@ static void __exit aesni_exit(void)
aesni_simd_aeads);
simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
aesni_simd_skciphers);
crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
crypto_unregister_alg(&aesni_cipher_alg);
}
late_initcall(aesni_init);

View File

@ -124,7 +124,7 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src,
}
static int chacha_simd_stream_xor(struct skcipher_walk *walk,
struct chacha_ctx *ctx, u8 *iv)
const struct chacha_ctx *ctx, const u8 *iv)
{
u32 *state, state_buf[16 + 2] __aligned(8);
int next_yield = 4096; /* bytes until next FPU yield */

View File

@ -61,7 +61,6 @@ config CRYPTO_BLKCIPHER2
tristate
select CRYPTO_ALGAPI2
select CRYPTO_RNG2
select CRYPTO_WORKQUEUE
config CRYPTO_HASH
tristate
@ -137,10 +136,11 @@ config CRYPTO_USER
Userspace configuration for cryptographic instantiations such as
cbc(aes).
if CRYPTO_MANAGER2
config CRYPTO_MANAGER_DISABLE_TESTS
bool "Disable run-time self tests"
default y
depends on CRYPTO_MANAGER2
help
Disable run-time self tests that normally take place at
algorithm registration.
@ -155,14 +155,10 @@ config CRYPTO_MANAGER_EXTRA_TESTS
This is intended for developer use only, as these tests take much
longer to run than the normal self tests.
endif # if CRYPTO_MANAGER2
config CRYPTO_GF128MUL
tristate "GF(2^128) multiplication functions"
help
Efficient table driven implementation of multiplications in the
field GF(2^128). This is needed by some cypher modes. This
option will be selected automatically if you select such a
cipher mode. Only select this option by hand if you expect to load
an external module that requires these functions.
tristate
config CRYPTO_NULL
tristate "Null algorithms"
@ -186,15 +182,11 @@ config CRYPTO_PCRYPT
This converts an arbitrary crypto algorithm into a parallel
algorithm that executes in kernel threads.
config CRYPTO_WORKQUEUE
tristate
config CRYPTO_CRYPTD
tristate "Software async crypto daemon"
select CRYPTO_BLKCIPHER
select CRYPTO_HASH
select CRYPTO_MANAGER
select CRYPTO_WORKQUEUE
help
This is a generic software asynchronous crypto daemon that
converts an arbitrary synchronous software crypto algorithm
@ -279,6 +271,7 @@ config CRYPTO_CCM
select CRYPTO_CTR
select CRYPTO_HASH
select CRYPTO_AEAD
select CRYPTO_MANAGER
help
Support for Counter with CBC MAC. Required for IPsec.
@ -288,6 +281,7 @@ config CRYPTO_GCM
select CRYPTO_AEAD
select CRYPTO_GHASH
select CRYPTO_NULL
select CRYPTO_MANAGER
help
Support for Galois/Counter Mode (GCM) and Galois Message
Authentication Code (GMAC). Required for IPSec.
@ -297,6 +291,7 @@ config CRYPTO_CHACHA20POLY1305
select CRYPTO_CHACHA20
select CRYPTO_POLY1305
select CRYPTO_AEAD
select CRYPTO_MANAGER
help
ChaCha20-Poly1305 AEAD support, RFC7539.
@ -411,6 +406,7 @@ config CRYPTO_SEQIV
select CRYPTO_BLKCIPHER
select CRYPTO_NULL
select CRYPTO_RNG_DEFAULT
select CRYPTO_MANAGER
help
This IV generator generates an IV based on a sequence number by
xoring it with a salt. This algorithm is mainly useful for CTR
@ -420,7 +416,7 @@ config CRYPTO_ECHAINIV
select CRYPTO_AEAD
select CRYPTO_NULL
select CRYPTO_RNG_DEFAULT
default m
select CRYPTO_MANAGER
help
This IV generator generates an IV based on the encryption of
a sequence number xored with a salt. This is the default
@ -456,6 +452,7 @@ config CRYPTO_CTR
config CRYPTO_CTS
tristate "CTS support"
select CRYPTO_BLKCIPHER
select CRYPTO_MANAGER
help
CTS: Cipher Text Stealing
This is the Cipher Text Stealing mode as described by
@ -521,6 +518,7 @@ config CRYPTO_XTS
config CRYPTO_KEYWRAP
tristate "Key wrapping support"
select CRYPTO_BLKCIPHER
select CRYPTO_MANAGER
help
Support for key wrapping (NIST SP800-38F / RFC3394) without
padding.
@ -551,6 +549,7 @@ config CRYPTO_ADIANTUM
select CRYPTO_CHACHA20
select CRYPTO_POLY1305
select CRYPTO_NHPOLY1305
select CRYPTO_MANAGER
help
Adiantum is a tweakable, length-preserving encryption mode
designed for fast and secure disk encryption, especially on
@ -684,6 +683,14 @@ config CRYPTO_CRC32_MIPS
instructions, when available.
config CRYPTO_XXHASH
tristate "xxHash hash algorithm"
select CRYPTO_HASH
select XXHASH
help
xxHash non-cryptographic hash algorithm. Extremely fast, working at
speeds close to RAM limits.
config CRYPTO_CRCT10DIF
tristate "CRCT10DIF algorithm"
select CRYPTO_HASH
@ -1230,9 +1237,13 @@ config CRYPTO_ANUBIS
<https://www.cosic.esat.kuleuven.be/nessie/reports/>
<http://www.larc.usp.br/~pbarreto/AnubisPage.html>
config CRYPTO_LIB_ARC4
tristate
config CRYPTO_ARC4
tristate "ARC4 cipher algorithm"
select CRYPTO_BLKCIPHER
select CRYPTO_LIB_ARC4
help
ARC4 cipher algorithm.

View File

@ -6,8 +6,6 @@
obj-$(CONFIG_CRYPTO) += crypto.o
crypto-y := api.o cipher.o compress.o memneq.o
obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o
obj-$(CONFIG_CRYPTO_ENGINE) += crypto_engine.o
obj-$(CONFIG_CRYPTO_FIPS) += fips.o
@ -131,6 +129,7 @@ obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o
obj-$(CONFIG_CRYPTO_LZ4) += lz4.o
obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o
obj-$(CONFIG_CRYPTO_XXHASH) += xxhash_generic.o
obj-$(CONFIG_CRYPTO_842) += 842.o
obj-$(CONFIG_CRYPTO_RNG2) += rng.o
obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o

View File

@ -84,6 +84,42 @@ int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
}
EXPORT_SYMBOL_GPL(crypto_aead_setauthsize);
int crypto_aead_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_alg *alg = aead->base.__crt_alg;
unsigned int cryptlen = req->cryptlen;
int ret;
crypto_stats_get(alg);
if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
ret = -ENOKEY;
else
ret = crypto_aead_alg(aead)->encrypt(req);
crypto_stats_aead_encrypt(cryptlen, alg, ret);
return ret;
}
EXPORT_SYMBOL_GPL(crypto_aead_encrypt);
int crypto_aead_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_alg *alg = aead->base.__crt_alg;
unsigned int cryptlen = req->cryptlen;
int ret;
crypto_stats_get(alg);
if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
ret = -ENOKEY;
else if (req->cryptlen < crypto_aead_authsize(aead))
ret = -EINVAL;
else
ret = crypto_aead_alg(aead)->decrypt(req);
crypto_stats_aead_decrypt(cryptlen, alg, ret);
return ret;
}
EXPORT_SYMBOL_GPL(crypto_aead_decrypt);
static void crypto_aead_exit_tfm(struct crypto_tfm *tfm)
{
struct crypto_aead *aead = __crypto_aead_cast(tfm);

View File

@ -21,23 +21,6 @@
static LIST_HEAD(crypto_template_list);
static inline int crypto_set_driver_name(struct crypto_alg *alg)
{
static const char suffix[] = "-generic";
char *driver_name = alg->cra_driver_name;
int len;
if (*driver_name)
return 0;
len = strlcpy(driver_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
if (len + sizeof(suffix) > CRYPTO_MAX_ALG_NAME)
return -ENAMETOOLONG;
memcpy(driver_name + len, suffix, sizeof(suffix));
return 0;
}
static inline void crypto_check_module_sig(struct module *mod)
{
if (fips_enabled && mod && !module_sig_ok(mod))
@ -49,6 +32,9 @@ static int crypto_check_alg(struct crypto_alg *alg)
{
crypto_check_module_sig(alg->cra_module);
if (!alg->cra_name[0] || !alg->cra_driver_name[0])
return -EINVAL;
if (alg->cra_alignmask & (alg->cra_alignmask + 1))
return -EINVAL;
@ -74,7 +60,7 @@ static int crypto_check_alg(struct crypto_alg *alg)
refcount_set(&alg->cra_refcnt, 1);
return crypto_set_driver_name(alg);
return 0;
}
static void crypto_free_instance(struct crypto_instance *inst)
@ -947,19 +933,6 @@ struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue)
}
EXPORT_SYMBOL_GPL(crypto_dequeue_request);
int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm)
{
struct crypto_async_request *req;
list_for_each_entry(req, &queue->list, list) {
if (req->tfm == tfm)
return 1;
}
return 0;
}
EXPORT_SYMBOL_GPL(crypto_tfm_in_queue);
static inline void crypto_inc_byte(u8 *a, unsigned int size)
{
u8 *b = (a + size);

View File

@ -673,6 +673,7 @@ static void anubis_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
static struct crypto_alg anubis_alg = {
.cra_name = "anubis",
.cra_driver_name = "anubis-generic",
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = ANUBIS_BLOCK_SIZE,
.cra_ctxsize = sizeof (struct anubis_ctx),

View File

@ -13,84 +13,15 @@
#include <linux/init.h>
#include <linux/module.h>
struct arc4_ctx {
u32 S[256];
u32 x, y;
};
static int arc4_set_key(struct crypto_tfm *tfm, const u8 *in_key,
static int crypto_arc4_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
{
struct arc4_ctx *ctx = crypto_tfm_ctx(tfm);
int i, j = 0, k = 0;
struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm);
ctx->x = 1;
ctx->y = 0;
for (i = 0; i < 256; i++)
ctx->S[i] = i;
for (i = 0; i < 256; i++) {
u32 a = ctx->S[i];
j = (j + in_key[k] + a) & 0xff;
ctx->S[i] = ctx->S[j];
ctx->S[j] = a;
if (++k >= key_len)
k = 0;
return arc4_setkey(ctx, in_key, key_len);
}
return 0;
}
static int arc4_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
{
return arc4_set_key(&tfm->base, in_key, key_len);
}
static void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in,
unsigned int len)
{
u32 *const S = ctx->S;
u32 x, y, a, b;
u32 ty, ta, tb;
if (len == 0)
return;
x = ctx->x;
y = ctx->y;
a = S[x];
y = (y + a) & 0xff;
b = S[y];
do {
S[y] = a;
a = (a + b) & 0xff;
S[x] = b;
x = (x + 1) & 0xff;
ta = S[x];
ty = (y + ta) & 0xff;
tb = S[ty];
*out++ = *in++ ^ S[a];
if (--len == 0)
break;
y = ty;
a = ta;
b = tb;
} while (true);
ctx->x = x;
ctx->y = y;
}
static void arc4_crypt_one(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
arc4_crypt(crypto_tfm_ctx(tfm), out, in, 1);
}
static int ecb_arc4_crypt(struct skcipher_request *req)
static int crypto_arc4_crypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm);
@ -108,54 +39,32 @@ static int ecb_arc4_crypt(struct skcipher_request *req)
return err;
}
static struct crypto_alg arc4_cipher = {
.cra_name = "arc4",
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = ARC4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct arc4_ctx),
.cra_module = THIS_MODULE,
.cra_u = {
.cipher = {
.cia_min_keysize = ARC4_MIN_KEY_SIZE,
.cia_max_keysize = ARC4_MAX_KEY_SIZE,
.cia_setkey = arc4_set_key,
.cia_encrypt = arc4_crypt_one,
.cia_decrypt = arc4_crypt_one,
},
},
};
static struct skcipher_alg arc4_skcipher = {
static struct skcipher_alg arc4_alg = {
/*
* For legacy reasons, this is named "ecb(arc4)", not "arc4".
* Nevertheless it's actually a stream cipher, not a block cipher.
*/
.base.cra_name = "ecb(arc4)",
.base.cra_driver_name = "ecb(arc4)-generic",
.base.cra_priority = 100,
.base.cra_blocksize = ARC4_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct arc4_ctx),
.base.cra_module = THIS_MODULE,
.min_keysize = ARC4_MIN_KEY_SIZE,
.max_keysize = ARC4_MAX_KEY_SIZE,
.setkey = arc4_set_key_skcipher,
.encrypt = ecb_arc4_crypt,
.decrypt = ecb_arc4_crypt,
.setkey = crypto_arc4_setkey,
.encrypt = crypto_arc4_crypt,
.decrypt = crypto_arc4_crypt,
};
static int __init arc4_init(void)
{
int err;
err = crypto_register_alg(&arc4_cipher);
if (err)
return err;
err = crypto_register_skcipher(&arc4_skcipher);
if (err)
crypto_unregister_alg(&arc4_cipher);
return err;
return crypto_register_skcipher(&arc4_alg);
}
static void __exit arc4_exit(void)
{
crypto_unregister_alg(&arc4_cipher);
crypto_unregister_skcipher(&arc4_skcipher);
crypto_unregister_skcipher(&arc4_alg);
}
subsys_initcall(arc4_init);
@ -164,4 +73,4 @@ module_exit(arc4_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("ARC4 Cipher Algorithm");
MODULE_AUTHOR("Jon Oberheide <jon@oberheide.org>");
MODULE_ALIAS_CRYPTO("arc4");
MODULE_ALIAS_CRYPTO("ecb(arc4)");

View File

@ -15,6 +15,7 @@ config ASYMMETRIC_PUBLIC_KEY_SUBTYPE
select MPILIB
select CRYPTO_HASH_INFO
select CRYPTO_AKCIPHER
select CRYPTO_HASH
help
This option provides support for asymmetric public key type handling.
If signature generation and/or verification are to be used,
@ -65,6 +66,7 @@ config TPM_KEY_PARSER
config PKCS7_MESSAGE_PARSER
tristate "PKCS#7 message parser"
depends on X509_CERTIFICATE_PARSER
select CRYPTO_HASH
select ASN1
select OID_REGISTRY
help
@ -87,6 +89,7 @@ config SIGNED_PE_FILE_VERIFICATION
bool "Support for PE file signature verification"
depends on PKCS7_MESSAGE_PARSER=y
depends on SYSTEM_DATA_VERIFICATION
select CRYPTO_HASH
select ASN1
select OID_REGISTRY
help

View File

@ -61,6 +61,8 @@ struct chachapoly_req_ctx {
unsigned int cryptlen;
/* Actual AD, excluding IV */
unsigned int assoclen;
/* request flags, with MAY_SLEEP cleared if needed */
u32 flags;
union {
struct poly_req poly;
struct chacha_req chacha;
@ -70,8 +72,12 @@ struct chachapoly_req_ctx {
static inline void async_done_continue(struct aead_request *req, int err,
int (*cont)(struct aead_request *))
{
if (!err)
if (!err) {
struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
err = cont(req);
}
if (err != -EINPROGRESS && err != -EBUSY)
aead_request_complete(req, err);
@ -129,16 +135,12 @@ static int chacha_decrypt(struct aead_request *req)
chacha_iv(creq->iv, req, 1);
sg_init_table(rctx->src, 2);
src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
dst = src;
if (req->src != req->dst) {
sg_init_table(rctx->dst, 2);
if (req->src != req->dst)
dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
}
skcipher_request_set_callback(&creq->req, aead_request_flags(req),
skcipher_request_set_callback(&creq->req, rctx->flags,
chacha_decrypt_done, req);
skcipher_request_set_tfm(&creq->req, ctx->chacha);
skcipher_request_set_crypt(&creq->req, src, dst,
@ -172,17 +174,13 @@ static int poly_tail(struct aead_request *req)
struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
struct poly_req *preq = &rctx->u.poly;
__le64 len;
int err;
sg_init_table(preq->src, 1);
len = cpu_to_le64(rctx->assoclen);
memcpy(&preq->tail.assoclen, &len, sizeof(len));
len = cpu_to_le64(rctx->cryptlen);
memcpy(&preq->tail.cryptlen, &len, sizeof(len));
sg_set_buf(preq->src, &preq->tail, sizeof(preq->tail));
preq->tail.assoclen = cpu_to_le64(rctx->assoclen);
preq->tail.cryptlen = cpu_to_le64(rctx->cryptlen);
sg_init_one(preq->src, &preq->tail, sizeof(preq->tail));
ahash_request_set_callback(&preq->req, aead_request_flags(req),
ahash_request_set_callback(&preq->req, rctx->flags,
poly_tail_done, req);
ahash_request_set_tfm(&preq->req, ctx->poly);
ahash_request_set_crypt(&preq->req, preq->src,
@ -205,15 +203,14 @@ static int poly_cipherpad(struct aead_request *req)
struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
struct poly_req *preq = &rctx->u.poly;
unsigned int padlen, bs = POLY1305_BLOCK_SIZE;
unsigned int padlen;
int err;
padlen = (bs - (rctx->cryptlen % bs)) % bs;
padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE;
memset(preq->pad, 0, sizeof(preq->pad));
sg_init_table(preq->src, 1);
sg_set_buf(preq->src, &preq->pad, padlen);
sg_init_one(preq->src, preq->pad, padlen);
ahash_request_set_callback(&preq->req, aead_request_flags(req),
ahash_request_set_callback(&preq->req, rctx->flags,
poly_cipherpad_done, req);
ahash_request_set_tfm(&preq->req, ctx->poly);
ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
@ -241,10 +238,9 @@ static int poly_cipher(struct aead_request *req)
if (rctx->cryptlen == req->cryptlen) /* encrypting */
crypt = req->dst;
sg_init_table(rctx->src, 2);
crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen);
ahash_request_set_callback(&preq->req, aead_request_flags(req),
ahash_request_set_callback(&preq->req, rctx->flags,
poly_cipher_done, req);
ahash_request_set_tfm(&preq->req, ctx->poly);
ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen);
@ -266,15 +262,14 @@ static int poly_adpad(struct aead_request *req)
struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
struct poly_req *preq = &rctx->u.poly;
unsigned int padlen, bs = POLY1305_BLOCK_SIZE;
unsigned int padlen;
int err;
padlen = (bs - (rctx->assoclen % bs)) % bs;
padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE;
memset(preq->pad, 0, sizeof(preq->pad));
sg_init_table(preq->src, 1);
sg_set_buf(preq->src, preq->pad, padlen);
sg_init_one(preq->src, preq->pad, padlen);
ahash_request_set_callback(&preq->req, aead_request_flags(req),
ahash_request_set_callback(&preq->req, rctx->flags,
poly_adpad_done, req);
ahash_request_set_tfm(&preq->req, ctx->poly);
ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
@ -298,7 +293,7 @@ static int poly_ad(struct aead_request *req)
struct poly_req *preq = &rctx->u.poly;
int err;
ahash_request_set_callback(&preq->req, aead_request_flags(req),
ahash_request_set_callback(&preq->req, rctx->flags,
poly_ad_done, req);
ahash_request_set_tfm(&preq->req, ctx->poly);
ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen);
@ -322,10 +317,9 @@ static int poly_setkey(struct aead_request *req)
struct poly_req *preq = &rctx->u.poly;
int err;
sg_init_table(preq->src, 1);
sg_set_buf(preq->src, rctx->key, sizeof(rctx->key));
sg_init_one(preq->src, rctx->key, sizeof(rctx->key));
ahash_request_set_callback(&preq->req, aead_request_flags(req),
ahash_request_set_callback(&preq->req, rctx->flags,
poly_setkey_done, req);
ahash_request_set_tfm(&preq->req, ctx->poly);
ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key));
@ -349,7 +343,7 @@ static int poly_init(struct aead_request *req)
struct poly_req *preq = &rctx->u.poly;
int err;
ahash_request_set_callback(&preq->req, aead_request_flags(req),
ahash_request_set_callback(&preq->req, rctx->flags,
poly_init_done, req);
ahash_request_set_tfm(&preq->req, ctx->poly);
@ -381,13 +375,12 @@ static int poly_genkey(struct aead_request *req)
rctx->assoclen -= 8;
}
sg_init_table(creq->src, 1);
memset(rctx->key, 0, sizeof(rctx->key));
sg_set_buf(creq->src, rctx->key, sizeof(rctx->key));
sg_init_one(creq->src, rctx->key, sizeof(rctx->key));
chacha_iv(creq->iv, req, 0);
skcipher_request_set_callback(&creq->req, aead_request_flags(req),
skcipher_request_set_callback(&creq->req, rctx->flags,
poly_genkey_done, req);
skcipher_request_set_tfm(&creq->req, ctx->chacha);
skcipher_request_set_crypt(&creq->req, creq->src, creq->src,
@ -418,16 +411,12 @@ static int chacha_encrypt(struct aead_request *req)
chacha_iv(creq->iv, req, 1);
sg_init_table(rctx->src, 2);
src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
dst = src;
if (req->src != req->dst) {
sg_init_table(rctx->dst, 2);
if (req->src != req->dst)
dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
}
skcipher_request_set_callback(&creq->req, aead_request_flags(req),
skcipher_request_set_callback(&creq->req, rctx->flags,
chacha_encrypt_done, req);
skcipher_request_set_tfm(&creq->req, ctx->chacha);
skcipher_request_set_crypt(&creq->req, src, dst,
@ -445,6 +434,7 @@ static int chachapoly_encrypt(struct aead_request *req)
struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
rctx->cryptlen = req->cryptlen;
rctx->flags = aead_request_flags(req);
/* encrypt call chain:
* - chacha_encrypt/done()
@ -466,6 +456,7 @@ static int chachapoly_decrypt(struct aead_request *req)
struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE;
rctx->flags = aead_request_flags(req);
/* decrypt call chain:
* - poly_genkey/done()

View File

@ -32,7 +32,7 @@ static void chacha_docrypt(u32 *state, u8 *dst, const u8 *src,
}
static int chacha_stream_xor(struct skcipher_request *req,
struct chacha_ctx *ctx, u8 *iv)
const struct chacha_ctx *ctx, const u8 *iv)
{
struct skcipher_walk walk;
u32 state[16];
@ -56,7 +56,7 @@ static int chacha_stream_xor(struct skcipher_request *req,
return err;
}
void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv)
void crypto_chacha_init(u32 *state, const struct chacha_ctx *ctx, const u8 *iv)
{
state[0] = 0x61707865; /* "expa" */
state[1] = 0x3320646e; /* "nd 3" */

View File

@ -16,7 +16,6 @@
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/cryptd.h>
#include <crypto/crypto_wq.h>
#include <linux/atomic.h>
#include <linux/err.h>
#include <linux/init.h>
@ -26,11 +25,14 @@
#include <linux/scatterlist.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
static unsigned int cryptd_max_cpu_qlen = 1000;
module_param(cryptd_max_cpu_qlen, uint, 0);
MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth");
static struct workqueue_struct *cryptd_wq;
struct cryptd_cpu_queue {
struct crypto_queue queue;
struct work_struct work;
@ -136,7 +138,7 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue,
if (err == -ENOSPC)
goto out_put_cpu;
queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
queue_work_on(cpu, cryptd_wq, &cpu_queue->work);
if (!atomic_read(refcnt))
goto out_put_cpu;
@ -179,7 +181,7 @@ static void cryptd_queue_worker(struct work_struct *work)
req->complete(req, 0);
if (cpu_queue->queue.qlen)
queue_work(kcrypto_wq, &cpu_queue->work);
queue_work(cryptd_wq, &cpu_queue->work);
}
static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
@ -919,7 +921,7 @@ static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
case CRYPTO_ALG_TYPE_BLKCIPHER:
return cryptd_create_skcipher(tmpl, tb, &queue);
case CRYPTO_ALG_TYPE_DIGEST:
case CRYPTO_ALG_TYPE_HASH:
return cryptd_create_hash(tmpl, tb, &queue);
case CRYPTO_ALG_TYPE_AEAD:
return cryptd_create_aead(tmpl, tb, &queue);
@ -1119,19 +1121,31 @@ static int __init cryptd_init(void)
{
int err;
cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE,
1);
if (!cryptd_wq)
return -ENOMEM;
err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen);
if (err)
return err;
goto err_destroy_wq;
err = crypto_register_template(&cryptd_tmpl);
if (err)
cryptd_fini_queue(&queue);
goto err_fini_queue;
return 0;
err_fini_queue:
cryptd_fini_queue(&queue);
err_destroy_wq:
destroy_workqueue(cryptd_wq);
return err;
}
static void __exit cryptd_exit(void)
{
destroy_workqueue(cryptd_wq);
cryptd_fini_queue(&queue);
crypto_unregister_template(&cryptd_tmpl);
}

View File

@ -100,6 +100,7 @@ static struct shash_alg digest_null = {
.final = null_final,
.base = {
.cra_name = "digest_null",
.cra_driver_name = "digest_null-generic",
.cra_blocksize = NULL_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
@ -122,6 +123,7 @@ static struct skcipher_alg skcipher_null = {
static struct crypto_alg null_algs[] = { {
.cra_name = "cipher_null",
.cra_driver_name = "cipher_null-generic",
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = NULL_BLOCK_SIZE,
.cra_ctxsize = 0,
@ -134,6 +136,7 @@ static struct crypto_alg null_algs[] = { {
.cia_decrypt = null_crypt } }
}, {
.cra_name = "compress_null",
.cra_driver_name = "compress_null-generic",
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_blocksize = NULL_BLOCK_SIZE,
.cra_ctxsize = 0,

View File

@ -1,35 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Workqueue for crypto subsystem
*
* Copyright (c) 2009 Intel Corp.
* Author: Huang Ying <ying.huang@intel.com>
*/
#include <linux/workqueue.h>
#include <linux/module.h>
#include <crypto/algapi.h>
#include <crypto/crypto_wq.h>
struct workqueue_struct *kcrypto_wq;
EXPORT_SYMBOL_GPL(kcrypto_wq);
static int __init crypto_wq_init(void)
{
kcrypto_wq = alloc_workqueue("crypto",
WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1);
if (unlikely(!kcrypto_wq))
return -ENOMEM;
return 0;
}
static void __exit crypto_wq_exit(void)
{
destroy_workqueue(kcrypto_wq);
}
subsys_initcall(crypto_wq_init);
module_exit(crypto_wq_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Workqueue for crypto subsystem");

View File

@ -275,6 +275,7 @@ static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src,
static struct crypto_alg alg = {
.cra_name = "deflate",
.cra_driver_name = "deflate-generic",
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_ctxsize = sizeof(struct deflate_ctx),
.cra_module = THIS_MODULE,

View File

@ -219,6 +219,57 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags)
}
}
/*
* FIPS 140-2 continuous self test for the noise source
* The test is performed on the noise source input data. Thus, the function
* implicitly knows the size of the buffer to be equal to the security
* strength.
*
* Note, this function disregards the nonce trailing the entropy data during
* initial seeding.
*
* drbg->drbg_mutex must have been taken.
*
* @drbg DRBG handle
* @entropy buffer of seed data to be checked
*
* return:
* 0 on success
* -EAGAIN on when the CTRNG is not yet primed
* < 0 on error
*/
static int drbg_fips_continuous_test(struct drbg_state *drbg,
const unsigned char *entropy)
{
unsigned short entropylen = drbg_sec_strength(drbg->core->flags);
int ret = 0;
if (!IS_ENABLED(CONFIG_CRYPTO_FIPS))
return 0;
/* skip test if we test the overall system */
if (list_empty(&drbg->test_data.list))
return 0;
/* only perform test in FIPS mode */
if (!fips_enabled)
return 0;
if (!drbg->fips_primed) {
/* Priming of FIPS test */
memcpy(drbg->prev, entropy, entropylen);
drbg->fips_primed = true;
/* priming: another round is needed */
return -EAGAIN;
}
ret = memcmp(drbg->prev, entropy, entropylen);
if (!ret)
panic("DRBG continuous self test failed\n");
memcpy(drbg->prev, entropy, entropylen);
/* the test shall pass when the two values are not equal */
return 0;
}
/*
* Convert an integer into a byte representation of this integer.
* The byte representation is big-endian
@ -998,6 +1049,22 @@ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed,
return ret;
}
static inline int drbg_get_random_bytes(struct drbg_state *drbg,
unsigned char *entropy,
unsigned int entropylen)
{
int ret;
do {
get_random_bytes(entropy, entropylen);
ret = drbg_fips_continuous_test(drbg, entropy);
if (ret && ret != -EAGAIN)
return ret;
} while (ret);
return 0;
}
static void drbg_async_seed(struct work_struct *work)
{
struct drbg_string data;
@ -1006,16 +1073,20 @@ static void drbg_async_seed(struct work_struct *work)
seed_work);
unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
unsigned char entropy[32];
int ret;
BUG_ON(!entropylen);
BUG_ON(entropylen > sizeof(entropy));
get_random_bytes(entropy, entropylen);
drbg_string_fill(&data, entropy, entropylen);
list_add_tail(&data.list, &seedlist);
mutex_lock(&drbg->drbg_mutex);
ret = drbg_get_random_bytes(drbg, entropy, entropylen);
if (ret)
goto unlock;
/* If nonblocking pool is initialized, deactivate Jitter RNG */
crypto_free_rng(drbg->jent);
drbg->jent = NULL;
@ -1030,6 +1101,7 @@ static void drbg_async_seed(struct work_struct *work)
if (drbg->seeded)
drbg->reseed_threshold = drbg_max_requests(drbg);
unlock:
mutex_unlock(&drbg->drbg_mutex);
memzero_explicit(entropy, entropylen);
@ -1081,7 +1153,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
BUG_ON((entropylen * 2) > sizeof(entropy));
/* Get seed from in-kernel /dev/urandom */
get_random_bytes(entropy, entropylen);
ret = drbg_get_random_bytes(drbg, entropy, entropylen);
if (ret)
goto out;
if (!drbg->jent) {
drbg_string_fill(&data1, entropy, entropylen);
@ -1094,7 +1168,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
entropylen);
if (ret) {
pr_devel("DRBG: jent failed with %d\n", ret);
return ret;
goto out;
}
drbg_string_fill(&data1, entropy, entropylen * 2);
@ -1121,6 +1195,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
ret = __drbg_seed(drbg, &seedlist, reseed);
out:
memzero_explicit(entropy, entropylen * 2);
return ret;
@ -1142,6 +1217,11 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg)
drbg->reseed_ctr = 0;
drbg->d_ops = NULL;
drbg->core = NULL;
if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) {
kzfree(drbg->prev);
drbg->prev = NULL;
drbg->fips_primed = false;
}
}
/*
@ -1211,6 +1291,14 @@ static inline int drbg_alloc_state(struct drbg_state *drbg)
drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1);
}
if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) {
drbg->prev = kzalloc(drbg_sec_strength(drbg->core->flags),
GFP_KERNEL);
if (!drbg->prev)
goto fini;
drbg->fips_primed = false;
}
return 0;
fini:

View File

@ -391,6 +391,7 @@ static int fcrypt_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key
static struct crypto_alg fcrypt_alg = {
.cra_name = "fcrypt",
.cra_driver_name = "fcrypt-generic",
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = 8,
.cra_ctxsize = sizeof(struct fcrypt_ctx),

View File

@ -31,6 +31,7 @@ static int ghash_setkey(struct crypto_shash *tfm,
const u8 *key, unsigned int keylen)
{
struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
be128 k;
if (keylen != GHASH_BLOCK_SIZE) {
crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
@ -39,7 +40,12 @@ static int ghash_setkey(struct crypto_shash *tfm,
if (ctx->gf128)
gf128mul_free_4k(ctx->gf128);
ctx->gf128 = gf128mul_init_4k_lle((be128 *)key);
BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE);
memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */
ctx->gf128 = gf128mul_init_4k_lle(&k);
memzero_explicit(&k, GHASH_BLOCK_SIZE);
if (!ctx->gf128)
return -ENOMEM;

View File

@ -56,11 +56,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector);
* Helper function
***************************************************************************/
__u64 jent_rol64(__u64 word, unsigned int shift)
{
return rol64(word, shift);
}
void *jent_zalloc(unsigned int len)
{
return kzalloc(len, GFP_KERNEL);

View File

@ -2,7 +2,7 @@
* Non-physical true random number generator based on timing jitter --
* Jitter RNG standalone code.
*
* Copyright Stephan Mueller <smueller@chronox.de>, 2015
* Copyright Stephan Mueller <smueller@chronox.de>, 2015 - 2019
*
* Design
* ======
@ -47,7 +47,7 @@
/*
* This Jitterentropy RNG is based on the jitterentropy library
* version 1.1.0 provided at http://www.chronox.de/jent.html
* version 2.1.2 provided at http://www.chronox.de/jent.html
*/
#ifdef __OPTIMIZE__
@ -71,10 +71,7 @@ struct rand_data {
#define DATA_SIZE_BITS ((sizeof(__u64)) * 8)
__u64 last_delta; /* SENSITIVE stuck test */
__s64 last_delta2; /* SENSITIVE stuck test */
unsigned int stuck:1; /* Time measurement stuck */
unsigned int osr; /* Oversample rate */
unsigned int stir:1; /* Post-processing stirring */
unsigned int disable_unbias:1; /* Deactivate Von-Neuman unbias */
#define JENT_MEMORY_BLOCKS 64
#define JENT_MEMORY_BLOCKSIZE 32
#define JENT_MEMORY_ACCESSLOOPS 128
@ -89,8 +86,6 @@ struct rand_data {
};
/* Flags that can be used to initialize the RNG */
#define JENT_DISABLE_STIR (1<<0) /* Disable stirring the entropy pool */
#define JENT_DISABLE_UNBIAS (1<<1) /* Disable the Von-Neuman Unbiaser */
#define JENT_DISABLE_MEMORY_ACCESS (1<<2) /* Disable memory access for more
* entropy, saves MEMORY_SIZE RAM for
* entropy collector */
@ -99,19 +94,16 @@ struct rand_data {
#define JENT_ENOTIME 1 /* Timer service not available */
#define JENT_ECOARSETIME 2 /* Timer too coarse for RNG */
#define JENT_ENOMONOTONIC 3 /* Timer is not monotonic increasing */
#define JENT_EMINVARIATION 4 /* Timer variations too small for RNG */
#define JENT_EVARVAR 5 /* Timer does not produce variations of
* variations (2nd derivation of time is
* zero). */
#define JENT_EMINVARVAR 6 /* Timer variations of variations is tooi
* small. */
#define JENT_ESTUCK 8 /* Too many stuck results during init. */
/***************************************************************************
* Helper functions
***************************************************************************/
void jent_get_nstime(__u64 *out);
__u64 jent_rol64(__u64 word, unsigned int shift);
void *jent_zalloc(unsigned int len);
void jent_zfree(void *ptr);
int jent_fips_enabled(void);
@ -140,16 +132,16 @@ static __u64 jent_loop_shuffle(struct rand_data *ec,
jent_get_nstime(&time);
/*
* mix the current state of the random number into the shuffle
* calculation to balance that shuffle a bit more
* Mix the current state of the random number into the shuffle
* calculation to balance that shuffle a bit more.
*/
if (ec)
time ^= ec->data;
/*
* we fold the time value as much as possible to ensure that as many
* bits of the time stamp are included as possible
* We fold the time value as much as possible to ensure that as many
* bits of the time stamp are included as possible.
*/
for (i = 0; (DATA_SIZE_BITS / bits) > i; i++) {
for (i = 0; ((DATA_SIZE_BITS + bits - 1) / bits) > i; i++) {
shuffle ^= time & mask;
time = time >> bits;
}
@ -169,38 +161,28 @@ static __u64 jent_loop_shuffle(struct rand_data *ec,
* CPU Jitter noise source -- this is the noise source based on the CPU
* execution time jitter
*
* This function folds the time into one bit units by iterating
* through the DATA_SIZE_BITS bit time value as follows: assume our time value
* is 0xabcd
* 1st loop, 1st shift generates 0xd000
* 1st loop, 2nd shift generates 0x000d
* 2nd loop, 1st shift generates 0xcd00
* 2nd loop, 2nd shift generates 0x000c
* 3rd loop, 1st shift generates 0xbcd0
* 3rd loop, 2nd shift generates 0x000b
* 4th loop, 1st shift generates 0xabcd
* 4th loop, 2nd shift generates 0x000a
* Now, the values at the end of the 2nd shifts are XORed together.
* This function injects the individual bits of the time value into the
* entropy pool using an LFSR.
*
* The code is deliberately inefficient and shall stay that way. This function
* is the root cause why the code shall be compiled without optimization. This
* function not only acts as folding operation, but this function's execution
* is used to measure the CPU execution time jitter. Any change to the loop in
* this function implies that careful retesting must be done.
* The code is deliberately inefficient with respect to the bit shifting
* and shall stay that way. This function is the root cause why the code
* shall be compiled without optimization. This function not only acts as
* folding operation, but this function's execution is used to measure
* the CPU execution time jitter. Any change to the loop in this function
* implies that careful retesting must be done.
*
* Input:
* @ec entropy collector struct -- may be NULL
* @time time stamp to be folded
* @time time stamp to be injected
* @loop_cnt if a value not equal to 0 is set, use the given value as number of
* loops to perform the folding
*
* Output:
* @folded result of folding operation
* updated ec->data
*
* @return Number of loops the folding operation is performed
*/
static __u64 jent_fold_time(struct rand_data *ec, __u64 time,
__u64 *folded, __u64 loop_cnt)
static __u64 jent_lfsr_time(struct rand_data *ec, __u64 time, __u64 loop_cnt)
{
unsigned int i;
__u64 j = 0;
@ -217,15 +199,34 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time,
if (loop_cnt)
fold_loop_cnt = loop_cnt;
for (j = 0; j < fold_loop_cnt; j++) {
new = 0;
new = ec->data;
for (i = 1; (DATA_SIZE_BITS) >= i; i++) {
__u64 tmp = time << (DATA_SIZE_BITS - i);
tmp = tmp >> (DATA_SIZE_BITS - 1);
/*
* Fibonacci LSFR with polynomial of
* x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is
* primitive according to
* http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf
* (the shift values are the polynomial values minus one
* due to counting bits from 0 to 63). As the current
* position is always the LSB, the polynomial only needs
* to shift data in from the left without wrap.
*/
tmp ^= ((new >> 63) & 1);
tmp ^= ((new >> 60) & 1);
tmp ^= ((new >> 55) & 1);
tmp ^= ((new >> 30) & 1);
tmp ^= ((new >> 27) & 1);
tmp ^= ((new >> 22) & 1);
new <<= 1;
new ^= tmp;
}
}
*folded = new;
ec->data = new;
return fold_loop_cnt;
}
@ -258,7 +259,6 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time,
*/
static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt)
{
unsigned char *tmpval = NULL;
unsigned int wrap = 0;
__u64 i = 0;
#define MAX_ACC_LOOP_BIT 7
@ -278,7 +278,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt)
acc_loop_cnt = loop_cnt;
for (i = 0; i < (ec->memaccessloops + acc_loop_cnt); i++) {
tmpval = ec->mem + ec->memlocation;
unsigned char *tmpval = ec->mem + ec->memlocation;
/*
* memory access: just add 1 to one byte,
* wrap at 255 -- memory access implies read
@ -316,7 +316,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt)
* 0 jitter measurement not stuck (good bit)
* 1 jitter measurement stuck (reject bit)
*/
static void jent_stuck(struct rand_data *ec, __u64 current_delta)
static int jent_stuck(struct rand_data *ec, __u64 current_delta)
{
__s64 delta2 = ec->last_delta - current_delta;
__s64 delta3 = delta2 - ec->last_delta2;
@ -325,14 +325,15 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta)
ec->last_delta2 = delta2;
if (!current_delta || !delta2 || !delta3)
ec->stuck = 1;
return 1;
return 0;
}
/**
* This is the heart of the entropy generation: calculate time deltas and
* use the CPU jitter in the time deltas. The jitter is folded into one
* bit. You can call this function the "random bit generator" as it
* produces one random bit per invocation.
* use the CPU jitter in the time deltas. The jitter is injected into the
* entropy pool.
*
* WARNING: ensure that ->prev_time is primed before using the output
* of this function! This can be done by calling this function
@ -341,12 +342,11 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta)
* Input:
* @entropy_collector Reference to entropy collector
*
* @return One random bit
* @return result of stuck test
*/
static __u64 jent_measure_jitter(struct rand_data *ec)
static int jent_measure_jitter(struct rand_data *ec)
{
__u64 time = 0;
__u64 data = 0;
__u64 current_delta = 0;
/* Invoke one noise source before time measurement to add variations */
@ -360,109 +360,11 @@ static __u64 jent_measure_jitter(struct rand_data *ec)
current_delta = time - ec->prev_time;
ec->prev_time = time;
/* Now call the next noise sources which also folds the data */
jent_fold_time(ec, current_delta, &data, 0);
/* Now call the next noise sources which also injects the data */
jent_lfsr_time(ec, current_delta, 0);
/*
* Check whether we have a stuck measurement. The enforcement
* is performed after the stuck value has been mixed into the
* entropy pool.
*/
jent_stuck(ec, current_delta);
return data;
}
/**
* Von Neuman unbias as explained in RFC 4086 section 4.2. As shown in the
* documentation of that RNG, the bits from jent_measure_jitter are considered
* independent which implies that the Von Neuman unbias operation is applicable.
* A proof of the Von-Neumann unbias operation to remove skews is given in the
* document "A proposal for: Functionality classes for random number
* generators", version 2.0 by Werner Schindler, section 5.4.1.
*
* Input:
* @entropy_collector Reference to entropy collector
*
* @return One random bit
*/
static __u64 jent_unbiased_bit(struct rand_data *entropy_collector)
{
do {
__u64 a = jent_measure_jitter(entropy_collector);
__u64 b = jent_measure_jitter(entropy_collector);
if (a == b)
continue;
if (1 == a)
return 1;
else
return 0;
} while (1);
}
/**
* Shuffle the pool a bit by mixing some value with a bijective function (XOR)
* into the pool.
*
* The function generates a mixer value that depends on the bits set and the
* location of the set bits in the random number generated by the entropy
* source. Therefore, based on the generated random number, this mixer value
* can have 2**64 different values. That mixer value is initialized with the
* first two SHA-1 constants. After obtaining the mixer value, it is XORed into
* the random number.
*
* The mixer value is not assumed to contain any entropy. But due to the XOR
* operation, it can also not destroy any entropy present in the entropy pool.
*
* Input:
* @entropy_collector Reference to entropy collector
*/
static void jent_stir_pool(struct rand_data *entropy_collector)
{
/*
* to shut up GCC on 32 bit, we have to initialize the 64 variable
* with two 32 bit variables
*/
union c {
__u64 u64;
__u32 u32[2];
};
/*
* This constant is derived from the first two 32 bit initialization
* vectors of SHA-1 as defined in FIPS 180-4 section 5.3.1
*/
union c constant;
/*
* The start value of the mixer variable is derived from the third
* and fourth 32 bit initialization vector of SHA-1 as defined in
* FIPS 180-4 section 5.3.1
*/
union c mixer;
unsigned int i = 0;
/*
* Store the SHA-1 constants in reverse order to make up the 64 bit
* value -- this applies to a little endian system, on a big endian
* system, it reverses as expected. But this really does not matter
* as we do not rely on the specific numbers. We just pick the SHA-1
* constants as they have a good mix of bit set and unset.
*/
constant.u32[1] = 0x67452301;
constant.u32[0] = 0xefcdab89;
mixer.u32[1] = 0x98badcfe;
mixer.u32[0] = 0x10325476;
for (i = 0; i < DATA_SIZE_BITS; i++) {
/*
* get the i-th bit of the input random number and only XOR
* the constant into the mixer value when that bit is set
*/
if ((entropy_collector->data >> i) & 1)
mixer.u64 ^= constant.u64;
mixer.u64 = jent_rol64(mixer.u64, 1);
}
entropy_collector->data ^= mixer.u64;
/* Check whether we have a stuck measurement. */
return jent_stuck(ec, current_delta);
}
/**
@ -480,48 +382,9 @@ static void jent_gen_entropy(struct rand_data *ec)
jent_measure_jitter(ec);
while (1) {
__u64 data = 0;
if (ec->disable_unbias == 1)
data = jent_measure_jitter(ec);
else
data = jent_unbiased_bit(ec);
/* enforcement of the jent_stuck test */
if (ec->stuck) {
/*
* We only mix in the bit considered not appropriate
* without the LSFR. The reason is that if we apply
* the LSFR and we do not rotate, the 2nd bit with LSFR
* will cancel out the first LSFR application on the
* bad bit.
*
* And we do not rotate as we apply the next bit to the
* current bit location again.
*/
ec->data ^= data;
ec->stuck = 0;
/* If a stuck measurement is received, repeat measurement */
if (jent_measure_jitter(ec))
continue;
}
/*
* Fibonacci LSFR with polynom of
* x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is
* primitive according to
* http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf
* (the shift values are the polynom values minus one
* due to counting bits from 0 to 63). As the current
* position is always the LSB, the polynom only needs
* to shift data in from the left without wrap.
*/
ec->data ^= data;
ec->data ^= ((ec->data >> 63) & 1);
ec->data ^= ((ec->data >> 60) & 1);
ec->data ^= ((ec->data >> 55) & 1);
ec->data ^= ((ec->data >> 30) & 1);
ec->data ^= ((ec->data >> 27) & 1);
ec->data ^= ((ec->data >> 22) & 1);
ec->data = jent_rol64(ec->data, 1);
/*
* We multiply the loop value with ->osr to obtain the
@ -530,8 +393,6 @@ static void jent_gen_entropy(struct rand_data *ec)
if (++k >= (DATA_SIZE_BITS * ec->osr))
break;
}
if (ec->stir)
jent_stir_pool(ec);
}
/**
@ -639,12 +500,6 @@ struct rand_data *jent_entropy_collector_alloc(unsigned int osr,
osr = 1; /* minimum sampling rate is 1 */
entropy_collector->osr = osr;
entropy_collector->stir = 1;
if (flags & JENT_DISABLE_STIR)
entropy_collector->stir = 0;
if (flags & JENT_DISABLE_UNBIAS)
entropy_collector->disable_unbias = 1;
/* fill the data pad with non-zero values */
jent_gen_entropy(entropy_collector);
@ -656,7 +511,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector)
jent_zfree(entropy_collector->mem);
entropy_collector->mem = NULL;
jent_zfree(entropy_collector);
entropy_collector = NULL;
}
int jent_entropy_init(void)
@ -665,8 +519,9 @@ int jent_entropy_init(void)
__u64 delta_sum = 0;
__u64 old_delta = 0;
int time_backwards = 0;
int count_var = 0;
int count_mod = 0;
int count_stuck = 0;
struct rand_data ec = { 0 };
/* We could perform statistical tests here, but the problem is
* that we only have a few loop counts to do testing. These
@ -695,12 +550,14 @@ int jent_entropy_init(void)
for (i = 0; (TESTLOOPCOUNT + CLEARCACHE) > i; i++) {
__u64 time = 0;
__u64 time2 = 0;
__u64 folded = 0;
__u64 delta = 0;
unsigned int lowdelta = 0;
int stuck;
/* Invoke core entropy collection logic */
jent_get_nstime(&time);
jent_fold_time(NULL, time, &folded, 1<<MIN_FOLD_LOOP_BIT);
ec.prev_time = time;
jent_lfsr_time(&ec, time, 0);
jent_get_nstime(&time2);
/* test whether timer works */
@ -715,6 +572,8 @@ int jent_entropy_init(void)
if (!delta)
return JENT_ECOARSETIME;
stuck = jent_stuck(&ec, delta);
/*
* up to here we did not modify any variable that will be
* evaluated later, but we already performed some work. Thus we
@ -725,14 +584,14 @@ int jent_entropy_init(void)
if (CLEARCACHE > i)
continue;
if (stuck)
count_stuck++;
/* test whether we have an increasing timer */
if (!(time2 > time))
time_backwards++;
/*
* Avoid modulo of 64 bit integer to allow code to compile
* on 32 bit architectures.
*/
/* use 32 bit value to ensure compilation on 32 bit arches */
lowdelta = time2 - time;
if (!(lowdelta % 100))
count_mod++;
@ -743,14 +602,10 @@ int jent_entropy_init(void)
* only after the first loop is executed as we need to prime
* the old_data value
*/
if (i) {
if (delta != old_delta)
count_var++;
if (delta > old_delta)
delta_sum += (delta - old_delta);
else
delta_sum += (old_delta - delta);
}
old_delta = delta;
}
@ -763,25 +618,29 @@ int jent_entropy_init(void)
*/
if (3 < time_backwards)
return JENT_ENOMONOTONIC;
/* Error if the time variances are always identical */
if (!delta_sum)
return JENT_EVARVAR;
/*
* Variations of deltas of time must on average be larger
* than 1 to ensure the entropy estimation
* implied with 1 is preserved
*/
if (delta_sum <= 1)
return JENT_EMINVARVAR;
if ((delta_sum) <= 1)
return JENT_EVARVAR;
/*
* Ensure that we have variations in the time stamp below 10 for at
* least 10% of all checks -- on some platforms, the counter
* increments in multiples of 100, but not always
* least 10% of all checks -- on some platforms, the counter increments
* in multiples of 100, but not always
*/
if ((TESTLOOPCOUNT/10 * 9) < count_mod)
return JENT_ECOARSETIME;
/*
* If we have more than 90% stuck results, then this Jitter RNG is
* likely to not work well.
*/
if ((TESTLOOPCOUNT/10 * 9) < count_stuck)
return JENT_ESTUCK;
return 0;
}

View File

@ -848,6 +848,7 @@ static void khazad_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
static struct crypto_alg khazad_alg = {
.cra_name = "khazad",
.cra_driver_name = "khazad-generic",
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = KHAZAD_BLOCK_SIZE,
.cra_ctxsize = sizeof (struct khazad_ctx),

View File

@ -384,7 +384,7 @@ static int create(struct crypto_template *tmpl, struct rtattr **tb)
inst->alg.base.cra_priority = alg->base.cra_priority;
inst->alg.base.cra_blocksize = LRW_BLOCK_SIZE;
inst->alg.base.cra_alignmask = alg->base.cra_alignmask |
(__alignof__(__be32) - 1);
(__alignof__(be128) - 1);
inst->alg.ivsize = LRW_BLOCK_SIZE;
inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) +

View File

@ -106,6 +106,7 @@ static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
static struct crypto_alg alg_lz4 = {
.cra_name = "lz4",
.cra_driver_name = "lz4-generic",
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_ctxsize = sizeof(struct lz4_ctx),
.cra_module = THIS_MODULE,

View File

@ -107,6 +107,7 @@ static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
static struct crypto_alg alg_lz4hc = {
.cra_name = "lz4hc",
.cra_driver_name = "lz4hc-generic",
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_ctxsize = sizeof(struct lz4hc_ctx),
.cra_module = THIS_MODULE,

View File

@ -109,6 +109,7 @@ static int lzorle_sdecompress(struct crypto_scomp *tfm, const u8 *src,
static struct crypto_alg alg = {
.cra_name = "lzo-rle",
.cra_driver_name = "lzo-rle-generic",
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_ctxsize = sizeof(struct lzorle_ctx),
.cra_module = THIS_MODULE,

View File

@ -109,6 +109,7 @@ static int lzo_sdecompress(struct crypto_scomp *tfm, const u8 *src,
static struct crypto_alg alg = {
.cra_name = "lzo",
.cra_driver_name = "lzo-generic",
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_ctxsize = sizeof(struct lzo_ctx),
.cra_module = THIS_MODULE,

View File

@ -217,6 +217,7 @@ static struct shash_alg alg = {
.descsize = sizeof(struct md4_ctx),
.base = {
.cra_name = "md4",
.cra_driver_name = "md4-generic",
.cra_blocksize = MD4_HMAC_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}

View File

@ -229,6 +229,7 @@ static struct shash_alg alg = {
.statesize = sizeof(struct md5_state),
.base = {
.cra_name = "md5",
.cra_driver_name = "md5-generic",
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}

View File

@ -156,6 +156,7 @@ static struct shash_alg alg = {
.descsize = sizeof(struct michael_mic_desc_ctx),
.base = {
.cra_name = "michael_mic",
.cra_driver_name = "michael_mic-generic",
.cra_blocksize = 8,
.cra_alignmask = 3,
.cra_ctxsize = sizeof(struct michael_mic_ctx),

View File

@ -298,6 +298,7 @@ static struct shash_alg alg = {
.descsize = sizeof(struct rmd128_ctx),
.base = {
.cra_name = "rmd128",
.cra_driver_name = "rmd128-generic",
.cra_blocksize = RMD128_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}

View File

@ -342,6 +342,7 @@ static struct shash_alg alg = {
.descsize = sizeof(struct rmd160_ctx),
.base = {
.cra_name = "rmd160",
.cra_driver_name = "rmd160-generic",
.cra_blocksize = RMD160_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}

View File

@ -317,6 +317,7 @@ static struct shash_alg alg = {
.descsize = sizeof(struct rmd256_ctx),
.base = {
.cra_name = "rmd256",
.cra_driver_name = "rmd256-generic",
.cra_blocksize = RMD256_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}

View File

@ -366,6 +366,7 @@ static struct shash_alg alg = {
.descsize = sizeof(struct rmd320_ctx),
.base = {
.cra_name = "rmd320",
.cra_driver_name = "rmd320-generic",
.cra_blocksize = RMD320_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}

View File

@ -225,7 +225,13 @@
x4 ^= x2; \
})
static void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, u32 r3, u32 r4, u32 *k)
/*
* both gcc and clang have misoptimized this function in the past,
* producing horrible object code from spilling temporary variables
* on the stack. Forcing this part out of line avoids that.
*/
static noinline void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2,
u32 r3, u32 r4, u32 *k)
{
k += 100;
S3(r3, r4, r0, r1, r2); store_and_load_keys(r1, r2, r4, r3, 28, 24);
@ -637,6 +643,7 @@ static struct crypto_alg srp_algs[2] = { {
.cia_decrypt = serpent_decrypt } }
}, {
.cra_name = "tnepres",
.cra_driver_name = "tnepres-generic",
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = SERPENT_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct serpent_ctx),

View File

@ -837,6 +837,40 @@ static int skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
return 0;
}
int crypto_skcipher_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_alg *alg = tfm->base.__crt_alg;
unsigned int cryptlen = req->cryptlen;
int ret;
crypto_stats_get(alg);
if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
ret = -ENOKEY;
else
ret = tfm->encrypt(req);
crypto_stats_skcipher_encrypt(cryptlen, ret, alg);
return ret;
}
EXPORT_SYMBOL_GPL(crypto_skcipher_encrypt);
int crypto_skcipher_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_alg *alg = tfm->base.__crt_alg;
unsigned int cryptlen = req->cryptlen;
int ret;
crypto_stats_get(alg);
if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
ret = -ENOKEY;
else
ret = tfm->decrypt(req);
crypto_stats_skcipher_decrypt(cryptlen, ret, alg);
return ret;
}
EXPORT_SYMBOL_GPL(crypto_skcipher_decrypt);
static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm)
{
struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm);

View File

@ -216,6 +216,7 @@ static void xeta_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
static struct crypto_alg tea_algs[3] = { {
.cra_name = "tea",
.cra_driver_name = "tea-generic",
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = TEA_BLOCK_SIZE,
.cra_ctxsize = sizeof (struct tea_ctx),
@ -229,6 +230,7 @@ static struct crypto_alg tea_algs[3] = { {
.cia_decrypt = tea_decrypt } }
}, {
.cra_name = "xtea",
.cra_driver_name = "xtea-generic",
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = XTEA_BLOCK_SIZE,
.cra_ctxsize = sizeof (struct xtea_ctx),
@ -242,6 +244,7 @@ static struct crypto_alg tea_algs[3] = { {
.cia_decrypt = xtea_decrypt } }
}, {
.cra_name = "xeta",
.cra_driver_name = "xeta-generic",
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = XTEA_BLOCK_SIZE,
.cra_ctxsize = sizeof (struct xtea_ctx),

View File

@ -1032,6 +1032,205 @@ static void crypto_reenable_simd_for_test(void)
}
#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
static int build_hash_sglist(struct test_sglist *tsgl,
const struct hash_testvec *vec,
const struct testvec_config *cfg,
unsigned int alignmask,
const struct test_sg_division *divs[XBUFSIZE])
{
struct kvec kv;
struct iov_iter input;
kv.iov_base = (void *)vec->plaintext;
kv.iov_len = vec->psize;
iov_iter_kvec(&input, WRITE, &kv, 1, vec->psize);
return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
&input, divs);
}
static int check_hash_result(const char *type,
const u8 *result, unsigned int digestsize,
const struct hash_testvec *vec,
const char *vec_name,
const char *driver,
const struct testvec_config *cfg)
{
if (memcmp(result, vec->digest, digestsize) != 0) {
pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
type, driver, vec_name, cfg->name);
return -EINVAL;
}
if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
type, driver, vec_name, cfg->name);
return -EOVERFLOW;
}
return 0;
}
static inline int check_shash_op(const char *op, int err,
const char *driver, const char *vec_name,
const struct testvec_config *cfg)
{
if (err)
pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
driver, op, err, vec_name, cfg->name);
return err;
}
static inline const void *sg_data(struct scatterlist *sg)
{
return page_address(sg_page(sg)) + sg->offset;
}
/* Test one hash test vector in one configuration, using the shash API */
static int test_shash_vec_cfg(const char *driver,
const struct hash_testvec *vec,
const char *vec_name,
const struct testvec_config *cfg,
struct shash_desc *desc,
struct test_sglist *tsgl,
u8 *hashstate)
{
struct crypto_shash *tfm = desc->tfm;
const unsigned int alignmask = crypto_shash_alignmask(tfm);
const unsigned int digestsize = crypto_shash_digestsize(tfm);
const unsigned int statesize = crypto_shash_statesize(tfm);
const struct test_sg_division *divs[XBUFSIZE];
unsigned int i;
u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
int err;
/* Set the key, if specified */
if (vec->ksize) {
err = crypto_shash_setkey(tfm, vec->key, vec->ksize);
if (err) {
if (err == vec->setkey_error)
return 0;
pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
driver, vec_name, vec->setkey_error, err,
crypto_shash_get_flags(tfm));
return err;
}
if (vec->setkey_error) {
pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
driver, vec_name, vec->setkey_error);
return -EINVAL;
}
}
/* Build the scatterlist for the source data */
err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
if (err) {
pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
driver, vec_name, cfg->name);
return err;
}
/* Do the actual hashing */
testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
vec->digest_error) {
/* Just using digest() */
if (tsgl->nents != 1)
return 0;
if (cfg->nosimd)
crypto_disable_simd_for_test();
err = crypto_shash_digest(desc, sg_data(&tsgl->sgl[0]),
tsgl->sgl[0].length, result);
if (cfg->nosimd)
crypto_reenable_simd_for_test();
if (err) {
if (err == vec->digest_error)
return 0;
pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
driver, vec_name, vec->digest_error, err,
cfg->name);
return err;
}
if (vec->digest_error) {
pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
driver, vec_name, vec->digest_error, cfg->name);
return -EINVAL;
}
goto result_ready;
}
/* Using init(), zero or more update(), then final() or finup() */
if (cfg->nosimd)
crypto_disable_simd_for_test();
err = crypto_shash_init(desc);
if (cfg->nosimd)
crypto_reenable_simd_for_test();
err = check_shash_op("init", err, driver, vec_name, cfg);
if (err)
return err;
for (i = 0; i < tsgl->nents; i++) {
if (i + 1 == tsgl->nents &&
cfg->finalization_type == FINALIZATION_TYPE_FINUP) {
if (divs[i]->nosimd)
crypto_disable_simd_for_test();
err = crypto_shash_finup(desc, sg_data(&tsgl->sgl[i]),
tsgl->sgl[i].length, result);
if (divs[i]->nosimd)
crypto_reenable_simd_for_test();
err = check_shash_op("finup", err, driver, vec_name,
cfg);
if (err)
return err;
goto result_ready;
}
if (divs[i]->nosimd)
crypto_disable_simd_for_test();
err = crypto_shash_update(desc, sg_data(&tsgl->sgl[i]),
tsgl->sgl[i].length);
if (divs[i]->nosimd)
crypto_reenable_simd_for_test();
err = check_shash_op("update", err, driver, vec_name, cfg);
if (err)
return err;
if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
/* Test ->export() and ->import() */
testmgr_poison(hashstate + statesize,
TESTMGR_POISON_LEN);
err = crypto_shash_export(desc, hashstate);
err = check_shash_op("export", err, driver, vec_name,
cfg);
if (err)
return err;
if (!testmgr_is_poison(hashstate + statesize,
TESTMGR_POISON_LEN)) {
pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
driver, vec_name, cfg->name);
return -EOVERFLOW;
}
testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
err = crypto_shash_import(desc, hashstate);
err = check_shash_op("import", err, driver, vec_name,
cfg);
if (err)
return err;
}
}
if (cfg->nosimd)
crypto_disable_simd_for_test();
err = crypto_shash_final(desc, result);
if (cfg->nosimd)
crypto_reenable_simd_for_test();
err = check_shash_op("final", err, driver, vec_name, cfg);
if (err)
return err;
result_ready:
return check_hash_result("shash", result, digestsize, vec, vec_name,
driver, cfg);
}
static int do_ahash_op(int (*op)(struct ahash_request *req),
struct ahash_request *req,
struct crypto_wait *wait, bool nosimd)
@ -1049,25 +1248,26 @@ static int do_ahash_op(int (*op)(struct ahash_request *req),
return crypto_wait_req(err, wait);
}
static int check_nonfinal_hash_op(const char *op, int err,
static int check_nonfinal_ahash_op(const char *op, int err,
u8 *result, unsigned int digestsize,
const char *driver, const char *vec_name,
const struct testvec_config *cfg)
{
if (err) {
pr_err("alg: hash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
driver, op, err, vec_name, cfg->name);
return err;
}
if (!testmgr_is_poison(result, digestsize)) {
pr_err("alg: hash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
driver, op, vec_name, cfg->name);
return -EINVAL;
}
return 0;
}
static int test_hash_vec_cfg(const char *driver,
/* Test one hash test vector in one configuration, using the ahash API */
static int test_ahash_vec_cfg(const char *driver,
const struct hash_testvec *vec,
const char *vec_name,
const struct testvec_config *cfg,
@ -1082,8 +1282,6 @@ static int test_hash_vec_cfg(const char *driver,
const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
const struct test_sg_division *divs[XBUFSIZE];
DECLARE_CRYPTO_WAIT(wait);
struct kvec _input;
struct iov_iter input;
unsigned int i;
struct scatterlist *pending_sgl;
unsigned int pending_len;
@ -1096,26 +1294,22 @@ static int test_hash_vec_cfg(const char *driver,
if (err) {
if (err == vec->setkey_error)
return 0;
pr_err("alg: hash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
driver, vec_name, vec->setkey_error, err,
crypto_ahash_get_flags(tfm));
return err;
}
if (vec->setkey_error) {
pr_err("alg: hash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
driver, vec_name, vec->setkey_error);
return -EINVAL;
}
}
/* Build the scatterlist for the source data */
_input.iov_base = (void *)vec->plaintext;
_input.iov_len = vec->psize;
iov_iter_kvec(&input, WRITE, &_input, 1, vec->psize);
err = build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
&input, divs);
err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
if (err) {
pr_err("alg: hash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
driver, vec_name, cfg->name);
return err;
}
@ -1135,13 +1329,13 @@ static int test_hash_vec_cfg(const char *driver,
if (err) {
if (err == vec->digest_error)
return 0;
pr_err("alg: hash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
driver, vec_name, vec->digest_error, err,
cfg->name);
return err;
}
if (vec->digest_error) {
pr_err("alg: hash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
driver, vec_name, vec->digest_error, cfg->name);
return -EINVAL;
}
@ -1153,7 +1347,7 @@ static int test_hash_vec_cfg(const char *driver,
ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
ahash_request_set_crypt(req, NULL, result, 0);
err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd);
err = check_nonfinal_hash_op("init", err, result, digestsize,
err = check_nonfinal_ahash_op("init", err, result, digestsize,
driver, vec_name, cfg);
if (err)
return err;
@ -1170,7 +1364,7 @@ static int test_hash_vec_cfg(const char *driver,
pending_len);
err = do_ahash_op(crypto_ahash_update, req, &wait,
divs[i]->nosimd);
err = check_nonfinal_hash_op("update", err,
err = check_nonfinal_ahash_op("update", err,
result, digestsize,
driver, vec_name, cfg);
if (err)
@ -1183,21 +1377,21 @@ static int test_hash_vec_cfg(const char *driver,
testmgr_poison(hashstate + statesize,
TESTMGR_POISON_LEN);
err = crypto_ahash_export(req, hashstate);
err = check_nonfinal_hash_op("export", err,
err = check_nonfinal_ahash_op("export", err,
result, digestsize,
driver, vec_name, cfg);
if (err)
return err;
if (!testmgr_is_poison(hashstate + statesize,
TESTMGR_POISON_LEN)) {
pr_err("alg: hash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
driver, vec_name, cfg->name);
return -EOVERFLOW;
}
testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
err = crypto_ahash_import(req, hashstate);
err = check_nonfinal_hash_op("import", err,
err = check_nonfinal_ahash_op("import", err,
result, digestsize,
driver, vec_name, cfg);
if (err)
@ -1213,13 +1407,13 @@ static int test_hash_vec_cfg(const char *driver,
if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {
/* finish with update() and final() */
err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd);
err = check_nonfinal_hash_op("update", err, result, digestsize,
err = check_nonfinal_ahash_op("update", err, result, digestsize,
driver, vec_name, cfg);
if (err)
return err;
err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd);
if (err) {
pr_err("alg: hash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
driver, err, vec_name, cfg->name);
return err;
}
@ -1227,31 +1421,49 @@ static int test_hash_vec_cfg(const char *driver,
/* finish with finup() */
err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd);
if (err) {
pr_err("alg: hash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
driver, err, vec_name, cfg->name);
return err;
}
}
result_ready:
/* Check that the algorithm produced the correct digest */
if (memcmp(result, vec->digest, digestsize) != 0) {
pr_err("alg: hash: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
driver, vec_name, cfg->name);
return -EINVAL;
}
if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
pr_err("alg: hash: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
driver, vec_name, cfg->name);
return -EOVERFLOW;
return check_hash_result("ahash", result, digestsize, vec, vec_name,
driver, cfg);
}
return 0;
static int test_hash_vec_cfg(const char *driver,
const struct hash_testvec *vec,
const char *vec_name,
const struct testvec_config *cfg,
struct ahash_request *req,
struct shash_desc *desc,
struct test_sglist *tsgl,
u8 *hashstate)
{
int err;
/*
* For algorithms implemented as "shash", most bugs will be detected by
* both the shash and ahash tests. Test the shash API first so that the
* failures involve less indirection, so are easier to debug.
*/
if (desc) {
err = test_shash_vec_cfg(driver, vec, vec_name, cfg, desc, tsgl,
hashstate);
if (err)
return err;
}
return test_ahash_vec_cfg(driver, vec, vec_name, cfg, req, tsgl,
hashstate);
}
static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
unsigned int vec_num, struct ahash_request *req,
struct test_sglist *tsgl, u8 *hashstate)
struct shash_desc *desc, struct test_sglist *tsgl,
u8 *hashstate)
{
char vec_name[16];
unsigned int i;
@ -1262,7 +1474,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {
err = test_hash_vec_cfg(driver, vec, vec_name,
&default_hash_testvec_configs[i],
req, tsgl, hashstate);
req, desc, tsgl, hashstate);
if (err)
return err;
}
@ -1276,9 +1488,10 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
generate_random_testvec_config(&cfg, cfgname,
sizeof(cfgname));
err = test_hash_vec_cfg(driver, vec, vec_name, &cfg,
req, tsgl, hashstate);
req, desc, tsgl, hashstate);
if (err)
return err;
cond_resched();
}
}
#endif
@ -1290,14 +1503,12 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
* Generate a hash test vector from the given implementation.
* Assumes the buffers in 'vec' were already allocated.
*/
static void generate_random_hash_testvec(struct crypto_shash *tfm,
static void generate_random_hash_testvec(struct shash_desc *desc,
struct hash_testvec *vec,
unsigned int maxkeysize,
unsigned int maxdatasize,
char *name, size_t max_namelen)
{
SHASH_DESC_ON_STACK(desc, tfm);
/* Data */
vec->psize = generate_random_length(maxdatasize);
generate_random_bytes((u8 *)vec->plaintext, vec->psize);
@ -1314,7 +1525,7 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm,
vec->ksize = 1 + (prandom_u32() % maxkeysize);
generate_random_bytes((u8 *)vec->key, vec->ksize);
vec->setkey_error = crypto_shash_setkey(tfm, vec->key,
vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key,
vec->ksize);
/* If the key couldn't be set, no need to continue to digest. */
if (vec->setkey_error)
@ -1322,7 +1533,6 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm,
}
/* Digest */
desc->tfm = tfm;
vec->digest_error = crypto_shash_digest(desc, vec->plaintext,
vec->psize, (u8 *)vec->digest);
done:
@ -1338,6 +1548,7 @@ static int test_hash_vs_generic_impl(const char *driver,
const char *generic_driver,
unsigned int maxkeysize,
struct ahash_request *req,
struct shash_desc *desc,
struct test_sglist *tsgl,
u8 *hashstate)
{
@ -1348,10 +1559,11 @@ static int test_hash_vs_generic_impl(const char *driver,
const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;
char _generic_driver[CRYPTO_MAX_ALG_NAME];
struct crypto_shash *generic_tfm = NULL;
struct shash_desc *generic_desc = NULL;
unsigned int i;
struct hash_testvec vec = { 0 };
char vec_name[64];
struct testvec_config cfg;
struct testvec_config *cfg;
char cfgname[TESTVEC_CONFIG_NAMELEN];
int err;
@ -1381,6 +1593,20 @@ static int test_hash_vs_generic_impl(const char *driver,
return err;
}
cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
if (!cfg) {
err = -ENOMEM;
goto out;
}
generic_desc = kzalloc(sizeof(*desc) +
crypto_shash_descsize(generic_tfm), GFP_KERNEL);
if (!generic_desc) {
err = -ENOMEM;
goto out;
}
generic_desc->tfm = generic_tfm;
/* Check the algorithm properties for consistency. */
if (digestsize != crypto_shash_digestsize(generic_tfm)) {
@ -1412,23 +1638,25 @@ static int test_hash_vs_generic_impl(const char *driver,
}
for (i = 0; i < fuzz_iterations * 8; i++) {
generate_random_hash_testvec(generic_tfm, &vec,
generate_random_hash_testvec(generic_desc, &vec,
maxkeysize, maxdatasize,
vec_name, sizeof(vec_name));
generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname));
generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg,
req, tsgl, hashstate);
err = test_hash_vec_cfg(driver, &vec, vec_name, cfg,
req, desc, tsgl, hashstate);
if (err)
goto out;
cond_resched();
}
err = 0;
out:
kfree(cfg);
kfree(vec.key);
kfree(vec.plaintext);
kfree(vec.digest);
crypto_free_shash(generic_tfm);
kzfree(generic_desc);
return err;
}
#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
@ -1436,6 +1664,7 @@ static int test_hash_vs_generic_impl(const char *driver,
const char *generic_driver,
unsigned int maxkeysize,
struct ahash_request *req,
struct shash_desc *desc,
struct test_sglist *tsgl,
u8 *hashstate)
{
@ -1443,26 +1672,67 @@ static int test_hash_vs_generic_impl(const char *driver,
}
#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
static int alloc_shash(const char *driver, u32 type, u32 mask,
struct crypto_shash **tfm_ret,
struct shash_desc **desc_ret)
{
struct crypto_shash *tfm;
struct shash_desc *desc;
tfm = crypto_alloc_shash(driver, type, mask);
if (IS_ERR(tfm)) {
if (PTR_ERR(tfm) == -ENOENT) {
/*
* This algorithm is only available through the ahash
* API, not the shash API, so skip the shash tests.
*/
return 0;
}
pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n",
driver, PTR_ERR(tfm));
return PTR_ERR(tfm);
}
desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL);
if (!desc) {
crypto_free_shash(tfm);
return -ENOMEM;
}
desc->tfm = tfm;
*tfm_ret = tfm;
*desc_ret = desc;
return 0;
}
static int __alg_test_hash(const struct hash_testvec *vecs,
unsigned int num_vecs, const char *driver,
u32 type, u32 mask,
const char *generic_driver, unsigned int maxkeysize)
{
struct crypto_ahash *tfm;
struct crypto_ahash *atfm = NULL;
struct ahash_request *req = NULL;
struct crypto_shash *stfm = NULL;
struct shash_desc *desc = NULL;
struct test_sglist *tsgl = NULL;
u8 *hashstate = NULL;
unsigned int statesize;
unsigned int i;
int err;
tfm = crypto_alloc_ahash(driver, type, mask);
if (IS_ERR(tfm)) {
/*
* Always test the ahash API. This works regardless of whether the
* algorithm is implemented as ahash or shash.
*/
atfm = crypto_alloc_ahash(driver, type, mask);
if (IS_ERR(atfm)) {
pr_err("alg: hash: failed to allocate transform for %s: %ld\n",
driver, PTR_ERR(tfm));
return PTR_ERR(tfm);
driver, PTR_ERR(atfm));
return PTR_ERR(atfm);
}
req = ahash_request_alloc(tfm, GFP_KERNEL);
req = ahash_request_alloc(atfm, GFP_KERNEL);
if (!req) {
pr_err("alg: hash: failed to allocate request for %s\n",
driver);
@ -1470,6 +1740,14 @@ static int __alg_test_hash(const struct hash_testvec *vecs,
goto out;
}
/*
* If available also test the shash API, to cover corner cases that may
* be missed by testing the ahash API only.
*/
err = alloc_shash(driver, type, mask, &stfm, &desc);
if (err)
goto out;
tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);
if (!tsgl || init_test_sglist(tsgl) != 0) {
pr_err("alg: hash: failed to allocate test buffers for %s\n",
@ -1480,8 +1758,10 @@ static int __alg_test_hash(const struct hash_testvec *vecs,
goto out;
}
hashstate = kmalloc(crypto_ahash_statesize(tfm) + TESTMGR_POISON_LEN,
GFP_KERNEL);
statesize = crypto_ahash_statesize(atfm);
if (stfm)
statesize = max(statesize, crypto_shash_statesize(stfm));
hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL);
if (!hashstate) {
pr_err("alg: hash: failed to allocate hash state buffer for %s\n",
driver);
@ -1490,20 +1770,24 @@ static int __alg_test_hash(const struct hash_testvec *vecs,
}
for (i = 0; i < num_vecs; i++) {
err = test_hash_vec(driver, &vecs[i], i, req, tsgl, hashstate);
err = test_hash_vec(driver, &vecs[i], i, req, desc, tsgl,
hashstate);
if (err)
goto out;
cond_resched();
}
err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req,
tsgl, hashstate);
desc, tsgl, hashstate);
out:
kfree(hashstate);
if (tsgl) {
destroy_test_sglist(tsgl);
kfree(tsgl);
}
kfree(desc);
crypto_free_shash(stfm);
ahash_request_free(req);
crypto_free_ahash(tfm);
crypto_free_ahash(atfm);
return err;
}
@ -1755,6 +2039,7 @@ static int test_aead_vec(const char *driver, int enc,
&cfg, req, tsgls);
if (err)
return err;
cond_resched();
}
}
#endif
@ -1864,7 +2149,7 @@ static int test_aead_vs_generic_impl(const char *driver,
unsigned int i;
struct aead_testvec vec = { 0 };
char vec_name[64];
struct testvec_config cfg;
struct testvec_config *cfg;
char cfgname[TESTVEC_CONFIG_NAMELEN];
int err;
@ -1894,6 +2179,12 @@ static int test_aead_vs_generic_impl(const char *driver,
return err;
}
cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
if (!cfg) {
err = -ENOMEM;
goto out;
}
generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL);
if (!generic_req) {
err = -ENOMEM;
@ -1948,13 +2239,13 @@ static int test_aead_vs_generic_impl(const char *driver,
generate_random_aead_testvec(generic_req, &vec,
maxkeysize, maxdatasize,
vec_name, sizeof(vec_name));
generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname));
generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, &cfg,
err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, cfg,
req, tsgls);
if (err)
goto out;
err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, &cfg,
err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, cfg,
req, tsgls);
if (err)
goto out;
@ -1962,6 +2253,7 @@ static int test_aead_vs_generic_impl(const char *driver,
}
err = 0;
out:
kfree(cfg);
kfree(vec.key);
kfree(vec.iv);
kfree(vec.assoc);
@ -1994,6 +2286,7 @@ static int test_aead(const char *driver, int enc,
tsgls);
if (err)
return err;
cond_resched();
}
return 0;
}
@ -2336,6 +2629,7 @@ static int test_skcipher_vec(const char *driver, int enc,
&cfg, req, tsgls);
if (err)
return err;
cond_resched();
}
}
#endif
@ -2409,7 +2703,7 @@ static int test_skcipher_vs_generic_impl(const char *driver,
unsigned int i;
struct cipher_testvec vec = { 0 };
char vec_name[64];
struct testvec_config cfg;
struct testvec_config *cfg;
char cfgname[TESTVEC_CONFIG_NAMELEN];
int err;
@ -2443,6 +2737,12 @@ static int test_skcipher_vs_generic_impl(const char *driver,
return err;
}
cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
if (!cfg) {
err = -ENOMEM;
goto out;
}
generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);
if (!generic_req) {
err = -ENOMEM;
@ -2490,20 +2790,21 @@ static int test_skcipher_vs_generic_impl(const char *driver,
for (i = 0; i < fuzz_iterations * 8; i++) {
generate_random_cipher_testvec(generic_req, &vec, maxdatasize,
vec_name, sizeof(vec_name));
generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname));
generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name,
&cfg, req, tsgls);
cfg, req, tsgls);
if (err)
goto out;
err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name,
&cfg, req, tsgls);
cfg, req, tsgls);
if (err)
goto out;
cond_resched();
}
err = 0;
out:
kfree(cfg);
kfree(vec.key);
kfree(vec.iv);
kfree(vec.ptext);
@ -2535,6 +2836,7 @@ static int test_skcipher(const char *driver, int enc,
tsgls);
if (err)
return err;
cond_resched();
}
return 0;
}
@ -4125,6 +4427,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "ecb(arc4)",
.generic_driver = "ecb(arc4)-generic",
.test = alg_test_skcipher,
.suite = {
.cipher = __VECS(arc4_tv_template)
@ -4789,6 +5092,13 @@ static const struct alg_test_desc alg_test_descs[] = {
.alg = "xts512(paes)",
.test = alg_test_null,
.fips_allowed = 1,
}, {
.alg = "xxhash64",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
.hash = __VECS(xxhash64_tv_template)
}
}, {
.alg = "zlib-deflate",
.test = alg_test_comp,

View File

@ -38,7 +38,7 @@ struct hash_testvec {
const char *key;
const char *plaintext;
const char *digest;
unsigned short psize;
unsigned int psize;
unsigned short ksize;
int setkey_error;
int digest_error;
@ -69,7 +69,7 @@ struct cipher_testvec {
const char *ctext;
unsigned char wk; /* weak key flag */
unsigned short klen;
unsigned short len;
unsigned int len;
bool fips_skip;
bool generates_iv;
int setkey_error;
@ -105,9 +105,9 @@ struct aead_testvec {
unsigned char novrfy;
unsigned char wk;
unsigned char klen;
unsigned short plen;
unsigned short clen;
unsigned short alen;
unsigned int plen;
unsigned int clen;
unsigned int alen;
int setkey_error;
int setauthsize_error;
int crypt_error;
@ -33382,6 +33382,112 @@ static const struct hash_testvec crc32c_tv_template[] = {
}
};
static const struct hash_testvec xxhash64_tv_template[] = {
{
.psize = 0,
.digest = "\x99\xe9\xd8\x51\x37\xdb\x46\xef",
},
{
.plaintext = "\x40",
.psize = 1,
.digest = "\x20\x5c\x91\xaa\x88\xeb\x59\xd0",
},
{
.plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d"
"\x88\xc7\x9a\x09\x1a\x9b",
.psize = 14,
.digest = "\xa8\xe8\x2b\xa9\x92\xa1\x37\x4a",
},
{
.plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d"
"\x88\xc7\x9a\x09\x1a\x9b\x42\xe0"
"\xd4\x38\xa5\x2a\x26\xa5\x19\x4b"
"\x57\x65\x7f\xad\xc3\x7d\xca\x40"
"\x31\x65\x05\xbb\x31\xae\x51\x11"
"\xa8\xc0\xb3\x28\x42\xeb\x3c\x46"
"\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e"
"\xbc\xe3\x88\x53\xca\x8f\xc8\xd9"
"\x41\x26\x7a\x3d\x21\xdb\x1a\x3c"
"\x01\x1d\xc9\xe9\xb7\x3a\x78\x67"
"\x57\x20\x94\xf1\x1e\xfd\xce\x39"
"\x99\x57\x69\x39\xa5\xd0\x8d\xd9"
"\x43\xfe\x1d\x66\x04\x3c\x27\x6a"
"\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56"
"\xa5\xb3\xec\xd9\x1f\x42\x65\x66"
"\xaa\xbf\x87\x9b\xc5\x41\x9c\x27"
"\x3f\x2f\xa9\x55\x93\x01\x27\x33"
"\x43\x99\x4d\x81\x85\xae\x82\x00"
"\x6c\xd0\xd1\xa3\x57\x18\x06\xcc"
"\xec\x72\xf7\x8e\x87\x2d\x1f\x5e"
"\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18"
"\x89\x76\xd3\x5e\xb5\x5a\xc0\x01"
"\xd2\xa1\x9a\x50\xe6\x08\xb4\x76"
"\x56\x4f\x0e\xbc\x54\xfc\x67\xe6"
"\xb9\xc0\x28\x4b\xb5\xc3\xff\x79"
"\x52\xea\xa1\x90\xc3\xaf\x08\x70"
"\x12\x02\x0c\xdb\x94\x00\x38\x95"
"\xed\xfd\x08\xf7\xe8\x04",
.psize = 222,
.digest = "\x41\xfc\xd4\x29\xfe\xe7\x85\x17",
},
{
.psize = 0,
.key = "\xb1\x79\x37\x9e\x00\x00\x00\x00",
.ksize = 8,
.digest = "\xef\x17\x9b\x92\xa2\xfd\x75\xac",
},
{
.plaintext = "\x40",
.psize = 1,
.key = "\xb1\x79\x37\x9e\x00\x00\x00\x00",
.ksize = 8,
.digest = "\xd1\x70\x4f\x14\x02\xc4\x9e\x71",
},
{
.plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d"
"\x88\xc7\x9a\x09\x1a\x9b",
.psize = 14,
.key = "\xb1\x79\x37\x9e\x00\x00\x00\x00",
.ksize = 8,
.digest = "\xa4\xcd\xfe\x8e\x37\xe2\x1c\x64"
},
{
.plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d"
"\x88\xc7\x9a\x09\x1a\x9b\x42\xe0"
"\xd4\x38\xa5\x2a\x26\xa5\x19\x4b"
"\x57\x65\x7f\xad\xc3\x7d\xca\x40"
"\x31\x65\x05\xbb\x31\xae\x51\x11"
"\xa8\xc0\xb3\x28\x42\xeb\x3c\x46"
"\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e"
"\xbc\xe3\x88\x53\xca\x8f\xc8\xd9"
"\x41\x26\x7a\x3d\x21\xdb\x1a\x3c"
"\x01\x1d\xc9\xe9\xb7\x3a\x78\x67"
"\x57\x20\x94\xf1\x1e\xfd\xce\x39"
"\x99\x57\x69\x39\xa5\xd0\x8d\xd9"
"\x43\xfe\x1d\x66\x04\x3c\x27\x6a"
"\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56"
"\xa5\xb3\xec\xd9\x1f\x42\x65\x66"
"\xaa\xbf\x87\x9b\xc5\x41\x9c\x27"
"\x3f\x2f\xa9\x55\x93\x01\x27\x33"
"\x43\x99\x4d\x81\x85\xae\x82\x00"
"\x6c\xd0\xd1\xa3\x57\x18\x06\xcc"
"\xec\x72\xf7\x8e\x87\x2d\x1f\x5e"
"\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18"
"\x89\x76\xd3\x5e\xb5\x5a\xc0\x01"
"\xd2\xa1\x9a\x50\xe6\x08\xb4\x76"
"\x56\x4f\x0e\xbc\x54\xfc\x67\xe6"
"\xb9\xc0\x28\x4b\xb5\xc3\xff\x79"
"\x52\xea\xa1\x90\xc3\xaf\x08\x70"
"\x12\x02\x0c\xdb\x94\x00\x38\x95"
"\xed\xfd\x08\xf7\xe8\x04",
.psize = 222,
.key = "\xb1\x79\x37\x9e\x00\x00\x00\x00",
.ksize = 8,
.digest = "\x58\xbc\x55\xf2\x42\x81\x5c\xf0"
},
};
static const struct comp_testvec lz4_comp_tv_template[] = {
{
.inlen = 255,

View File

@ -631,6 +631,7 @@ static struct shash_alg tgr_algs[3] = { {
.descsize = sizeof(struct tgr192_ctx),
.base = {
.cra_name = "tgr192",
.cra_driver_name = "tgr192-generic",
.cra_blocksize = TGR192_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
@ -642,6 +643,7 @@ static struct shash_alg tgr_algs[3] = { {
.descsize = sizeof(struct tgr192_ctx),
.base = {
.cra_name = "tgr160",
.cra_driver_name = "tgr160-generic",
.cra_blocksize = TGR192_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
@ -653,6 +655,7 @@ static struct shash_alg tgr_algs[3] = { {
.descsize = sizeof(struct tgr192_ctx),
.base = {
.cra_name = "tgr128",
.cra_driver_name = "tgr128-generic",
.cra_blocksize = TGR192_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}

View File

@ -1127,6 +1127,7 @@ static struct shash_alg wp_algs[3] = { {
.descsize = sizeof(struct wp512_ctx),
.base = {
.cra_name = "wp512",
.cra_driver_name = "wp512-generic",
.cra_blocksize = WP512_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
@ -1138,6 +1139,7 @@ static struct shash_alg wp_algs[3] = { {
.descsize = sizeof(struct wp512_ctx),
.base = {
.cra_name = "wp384",
.cra_driver_name = "wp384-generic",
.cra_blocksize = WP512_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
@ -1149,6 +1151,7 @@ static struct shash_alg wp_algs[3] = { {
.descsize = sizeof(struct wp512_ctx),
.base = {
.cra_name = "wp256",
.cra_driver_name = "wp256-generic",
.cra_blocksize = WP512_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}

108
crypto/xxhash_generic.c Normal file
View File

@ -0,0 +1,108 @@
// SPDX-License-Identifier: GPL-2.0
#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/xxhash.h>
#include <asm/unaligned.h>
#define XXHASH64_BLOCK_SIZE 32
#define XXHASH64_DIGEST_SIZE 8
struct xxhash64_tfm_ctx {
u64 seed;
};
struct xxhash64_desc_ctx {
struct xxh64_state xxhstate;
};
static int xxhash64_setkey(struct crypto_shash *tfm, const u8 *key,
unsigned int keylen)
{
struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(tfm);
if (keylen != sizeof(tctx->seed)) {
crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
tctx->seed = get_unaligned_le64(key);
return 0;
}
static int xxhash64_init(struct shash_desc *desc)
{
struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc);
xxh64_reset(&dctx->xxhstate, tctx->seed);
return 0;
}
static int xxhash64_update(struct shash_desc *desc, const u8 *data,
unsigned int length)
{
struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc);
xxh64_update(&dctx->xxhstate, data, length);
return 0;
}
static int xxhash64_final(struct shash_desc *desc, u8 *out)
{
struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc);
put_unaligned_le64(xxh64_digest(&dctx->xxhstate), out);
return 0;
}
static int xxhash64_digest(struct shash_desc *desc, const u8 *data,
unsigned int length, u8 *out)
{
struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
put_unaligned_le64(xxh64(data, length, tctx->seed), out);
return 0;
}
static struct shash_alg alg = {
.digestsize = XXHASH64_DIGEST_SIZE,
.setkey = xxhash64_setkey,
.init = xxhash64_init,
.update = xxhash64_update,
.final = xxhash64_final,
.digest = xxhash64_digest,
.descsize = sizeof(struct xxhash64_desc_ctx),
.base = {
.cra_name = "xxhash64",
.cra_driver_name = "xxhash64-generic",
.cra_priority = 100,
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.cra_blocksize = XXHASH64_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct xxhash64_tfm_ctx),
.cra_module = THIS_MODULE,
}
};
static int __init xxhash_mod_init(void)
{
return crypto_register_shash(&alg);
}
static void __exit xxhash_mod_fini(void)
{
crypto_unregister_shash(&alg);
}
subsys_initcall(xxhash_mod_init);
module_exit(xxhash_mod_fini);
MODULE_AUTHOR("Nikolay Borisov <nborisov@suse.com>");
MODULE_DESCRIPTION("xxhash calculations wrapper for lib/xxhash.c");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("xxhash64");
MODULE_ALIAS_CRYPTO("xxhash64-generic");

View File

@ -206,6 +206,7 @@ static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src,
static struct crypto_alg alg = {
.cra_name = "zstd",
.cra_driver_name = "zstd-generic",
.cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
.cra_ctxsize = sizeof(struct zstd_ctx),
.cra_module = THIS_MODULE,

View File

@ -220,6 +220,7 @@ static int iproc_rng200_probe(struct platform_device *pdev)
}
static const struct of_device_id iproc_rng200_of_match[] = {
{ .compatible = "brcm,bcm7211-rng200", },
{ .compatible = "brcm,bcm7278-rng200", },
{ .compatible = "brcm,iproc-rng200", },
{},

View File

@ -1,58 +1,8 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* Copyright (c) 2016 BayLibre, SAS.
* Author: Neil Armstrong <narmstrong@baylibre.com>
* Copyright (C) 2014 Amlogic, Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
* The full GNU General Public License is included in this distribution
* in the file called COPYING.
*
* BSD LICENSE
*
* Copyright (c) 2016 BayLibre, SAS.
* Author: Neil Armstrong <narmstrong@baylibre.com>
* Copyright (C) 2014 Amlogic, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <linux/err.h>
#include <linux/module.h>

View File

@ -520,10 +520,13 @@ config CRYPTO_DEV_ATMEL_SHA
To compile this driver as a module, choose M here: the module
will be called atmel-sha.
config CRYPTO_DEV_ATMEL_I2C
tristate
config CRYPTO_DEV_ATMEL_ECC
tristate "Support for Microchip / Atmel ECC hw accelerator"
depends on ARCH_AT91 || COMPILE_TEST
depends on I2C
select CRYPTO_DEV_ATMEL_I2C
select CRYPTO_ECDH
select CRC16
help
@ -534,6 +537,21 @@ config CRYPTO_DEV_ATMEL_ECC
To compile this driver as a module, choose M here: the module
will be called atmel-ecc.
config CRYPTO_DEV_ATMEL_SHA204A
tristate "Support for Microchip / Atmel SHA accelerator and RNG"
depends on I2C
select CRYPTO_DEV_ATMEL_I2C
select HW_RANDOM
select CRC16
help
Microhip / Atmel SHA accelerator and RNG.
Select this if you want to use the Microchip / Atmel SHA204A
module as a random number generator. (Other functions of the
chip are currently not exposed by this driver)
To compile this driver as a module, choose M here: the module
will be called atmel-sha204a.
config CRYPTO_DEV_CCP
bool "Support for AMD Secure Processor"
depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM

View File

@ -2,7 +2,9 @@
obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o
obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/
obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/

View File

@ -67,12 +67,16 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm,
}
static inline int crypto4xx_crypt(struct skcipher_request *req,
const unsigned int ivlen, bool decrypt)
const unsigned int ivlen, bool decrypt,
bool check_blocksize)
{
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
__le32 iv[AES_IV_SIZE];
if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE))
return -EINVAL;
if (ivlen)
crypto4xx_memcpy_to_le32(iv, req->iv, ivlen);
@ -81,24 +85,34 @@ static inline int crypto4xx_crypt(struct skcipher_request *req,
ctx->sa_len, 0, NULL);
}
int crypto4xx_encrypt_noiv(struct skcipher_request *req)
int crypto4xx_encrypt_noiv_block(struct skcipher_request *req)
{
return crypto4xx_crypt(req, 0, false);
return crypto4xx_crypt(req, 0, false, true);
}
int crypto4xx_encrypt_iv(struct skcipher_request *req)
int crypto4xx_encrypt_iv_stream(struct skcipher_request *req)
{
return crypto4xx_crypt(req, AES_IV_SIZE, false);
return crypto4xx_crypt(req, AES_IV_SIZE, false, false);
}
int crypto4xx_decrypt_noiv(struct skcipher_request *req)
int crypto4xx_decrypt_noiv_block(struct skcipher_request *req)
{
return crypto4xx_crypt(req, 0, true);
return crypto4xx_crypt(req, 0, true, true);
}
int crypto4xx_decrypt_iv(struct skcipher_request *req)
int crypto4xx_decrypt_iv_stream(struct skcipher_request *req)
{
return crypto4xx_crypt(req, AES_IV_SIZE, true);
return crypto4xx_crypt(req, AES_IV_SIZE, true, false);
}
int crypto4xx_encrypt_iv_block(struct skcipher_request *req)
{
return crypto4xx_crypt(req, AES_IV_SIZE, false, true);
}
int crypto4xx_decrypt_iv_block(struct skcipher_request *req)
{
return crypto4xx_crypt(req, AES_IV_SIZE, true, true);
}
/**
@ -269,8 +283,8 @@ crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt)
return ret;
}
return encrypt ? crypto4xx_encrypt_iv(req)
: crypto4xx_decrypt_iv(req);
return encrypt ? crypto4xx_encrypt_iv_stream(req)
: crypto4xx_decrypt_iv_stream(req);
}
static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx,

View File

@ -182,7 +182,6 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev)
dev->pdr_pa);
return -ENOMEM;
}
memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD);
dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device,
sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD,
&dev->shadow_sa_pool_pa,
@ -1210,8 +1209,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_IV_SIZE,
.setkey = crypto4xx_setkey_aes_cbc,
.encrypt = crypto4xx_encrypt_iv,
.decrypt = crypto4xx_decrypt_iv,
.encrypt = crypto4xx_encrypt_iv_block,
.decrypt = crypto4xx_decrypt_iv_block,
.init = crypto4xx_sk_init,
.exit = crypto4xx_sk_exit,
} },
@ -1222,7 +1221,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
.cra_module = THIS_MODULE,
},
@ -1230,8 +1229,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_IV_SIZE,
.setkey = crypto4xx_setkey_aes_cfb,
.encrypt = crypto4xx_encrypt_iv,
.decrypt = crypto4xx_decrypt_iv,
.encrypt = crypto4xx_encrypt_iv_stream,
.decrypt = crypto4xx_decrypt_iv_stream,
.init = crypto4xx_sk_init,
.exit = crypto4xx_sk_exit,
} },
@ -1243,7 +1242,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
.cra_module = THIS_MODULE,
},
@ -1263,7 +1262,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
.cra_module = THIS_MODULE,
},
@ -1290,8 +1289,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = crypto4xx_setkey_aes_ecb,
.encrypt = crypto4xx_encrypt_noiv,
.decrypt = crypto4xx_decrypt_noiv,
.encrypt = crypto4xx_encrypt_noiv_block,
.decrypt = crypto4xx_decrypt_noiv_block,
.init = crypto4xx_sk_init,
.exit = crypto4xx_sk_exit,
} },
@ -1302,7 +1301,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
.cra_module = THIS_MODULE,
},
@ -1310,8 +1309,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_IV_SIZE,
.setkey = crypto4xx_setkey_aes_ofb,
.encrypt = crypto4xx_encrypt_iv,
.decrypt = crypto4xx_decrypt_iv,
.encrypt = crypto4xx_encrypt_iv_stream,
.decrypt = crypto4xx_decrypt_iv_stream,
.init = crypto4xx_sk_init,
.exit = crypto4xx_sk_exit,
} },

View File

@ -173,10 +173,12 @@ int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher,
const u8 *key, unsigned int keylen);
int crypto4xx_encrypt_ctr(struct skcipher_request *req);
int crypto4xx_decrypt_ctr(struct skcipher_request *req);
int crypto4xx_encrypt_iv(struct skcipher_request *req);
int crypto4xx_decrypt_iv(struct skcipher_request *req);
int crypto4xx_encrypt_noiv(struct skcipher_request *req);
int crypto4xx_decrypt_noiv(struct skcipher_request *req);
int crypto4xx_encrypt_iv_stream(struct skcipher_request *req);
int crypto4xx_decrypt_iv_stream(struct skcipher_request *req);
int crypto4xx_encrypt_iv_block(struct skcipher_request *req);
int crypto4xx_decrypt_iv_block(struct skcipher_request *req);
int crypto4xx_encrypt_noiv_block(struct skcipher_request *req);
int crypto4xx_decrypt_noiv_block(struct skcipher_request *req);
int crypto4xx_rfc3686_encrypt(struct skcipher_request *req);
int crypto4xx_rfc3686_decrypt(struct skcipher_request *req);
int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm);

View File

@ -6,8 +6,6 @@
* Author: Tudor Ambarus <tudor.ambarus@microchip.com>
*/
#include <linux/bitrev.h>
#include <linux/crc16.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/err.h>
@ -23,41 +21,10 @@
#include <crypto/internal/kpp.h>
#include <crypto/ecdh.h>
#include <crypto/kpp.h>
#include "atmel-ecc.h"
/* Used for binding tfm objects to i2c clients. */
struct atmel_ecc_driver_data {
struct list_head i2c_client_list;
spinlock_t i2c_list_lock;
} ____cacheline_aligned;
#include "atmel-i2c.h"
static struct atmel_ecc_driver_data driver_data;
/**
* atmel_ecc_i2c_client_priv - i2c_client private data
* @client : pointer to i2c client device
* @i2c_client_list_node: part of i2c_client_list
* @lock : lock for sending i2c commands
* @wake_token : wake token array of zeros
* @wake_token_sz : size in bytes of the wake_token
* @tfm_count : number of active crypto transformations on i2c client
*
* Reads and writes from/to the i2c client are sequential. The first byte
* transmitted to the device is treated as the byte size. Any attempt to send
* more than this number of bytes will cause the device to not ACK those bytes.
* After the host writes a single command byte to the input buffer, reads are
* prohibited until after the device completes command execution. Use a mutex
* when sending i2c commands.
*/
struct atmel_ecc_i2c_client_priv {
struct i2c_client *client;
struct list_head i2c_client_list_node;
struct mutex lock;
u8 wake_token[WAKE_TOKEN_MAX_SIZE];
size_t wake_token_sz;
atomic_t tfm_count ____cacheline_aligned;
};
/**
* atmel_ecdh_ctx - transformation context
* @client : pointer to i2c client device
@ -80,188 +47,12 @@ struct atmel_ecdh_ctx {
bool do_fallback;
};
/**
* atmel_ecc_work_data - data structure representing the work
* @ctx : transformation context.
* @cbk : pointer to a callback function to be invoked upon completion of this
* request. This has the form:
* callback(struct atmel_ecc_work_data *work_data, void *areq, u8 status)
* where:
* @work_data: data structure representing the work
* @areq : optional pointer to an argument passed with the original
* request.
* @status : status returned from the i2c client device or i2c error.
* @areq: optional pointer to a user argument for use at callback time.
* @work: describes the task to be executed.
* @cmd : structure used for communicating with the device.
*/
struct atmel_ecc_work_data {
struct atmel_ecdh_ctx *ctx;
void (*cbk)(struct atmel_ecc_work_data *work_data, void *areq,
int status);
void *areq;
struct work_struct work;
struct atmel_ecc_cmd cmd;
};
static u16 atmel_ecc_crc16(u16 crc, const u8 *buffer, size_t len)
{
return cpu_to_le16(bitrev16(crc16(crc, buffer, len)));
}
/**
* atmel_ecc_checksum() - Generate 16-bit CRC as required by ATMEL ECC.
* CRC16 verification of the count, opcode, param1, param2 and data bytes.
* The checksum is saved in little-endian format in the least significant
* two bytes of the command. CRC polynomial is 0x8005 and the initial register
* value should be zero.
*
* @cmd : structure used for communicating with the device.
*/
static void atmel_ecc_checksum(struct atmel_ecc_cmd *cmd)
{
u8 *data = &cmd->count;
size_t len = cmd->count - CRC_SIZE;
u16 *crc16 = (u16 *)(data + len);
*crc16 = atmel_ecc_crc16(0, data, len);
}
static void atmel_ecc_init_read_cmd(struct atmel_ecc_cmd *cmd)
{
cmd->word_addr = COMMAND;
cmd->opcode = OPCODE_READ;
/*
* Read the word from Configuration zone that contains the lock bytes
* (UserExtra, Selector, LockValue, LockConfig).
*/
cmd->param1 = CONFIG_ZONE;
cmd->param2 = DEVICE_LOCK_ADDR;
cmd->count = READ_COUNT;
atmel_ecc_checksum(cmd);
cmd->msecs = MAX_EXEC_TIME_READ;
cmd->rxsize = READ_RSP_SIZE;
}
static void atmel_ecc_init_genkey_cmd(struct atmel_ecc_cmd *cmd, u16 keyid)
{
cmd->word_addr = COMMAND;
cmd->count = GENKEY_COUNT;
cmd->opcode = OPCODE_GENKEY;
cmd->param1 = GENKEY_MODE_PRIVATE;
/* a random private key will be generated and stored in slot keyID */
cmd->param2 = cpu_to_le16(keyid);
atmel_ecc_checksum(cmd);
cmd->msecs = MAX_EXEC_TIME_GENKEY;
cmd->rxsize = GENKEY_RSP_SIZE;
}
static int atmel_ecc_init_ecdh_cmd(struct atmel_ecc_cmd *cmd,
struct scatterlist *pubkey)
{
size_t copied;
cmd->word_addr = COMMAND;
cmd->count = ECDH_COUNT;
cmd->opcode = OPCODE_ECDH;
cmd->param1 = ECDH_PREFIX_MODE;
/* private key slot */
cmd->param2 = cpu_to_le16(DATA_SLOT_2);
/*
* The device only supports NIST P256 ECC keys. The public key size will
* always be the same. Use a macro for the key size to avoid unnecessary
* computations.
*/
copied = sg_copy_to_buffer(pubkey,
sg_nents_for_len(pubkey,
ATMEL_ECC_PUBKEY_SIZE),
cmd->data, ATMEL_ECC_PUBKEY_SIZE);
if (copied != ATMEL_ECC_PUBKEY_SIZE)
return -EINVAL;
atmel_ecc_checksum(cmd);
cmd->msecs = MAX_EXEC_TIME_ECDH;
cmd->rxsize = ECDH_RSP_SIZE;
return 0;
}
/*
* After wake and after execution of a command, there will be error, status, or
* result bytes in the device's output register that can be retrieved by the
* system. When the length of that group is four bytes, the codes returned are
* detailed in error_list.
*/
static int atmel_ecc_status(struct device *dev, u8 *status)
{
size_t err_list_len = ARRAY_SIZE(error_list);
int i;
u8 err_id = status[1];
if (*status != STATUS_SIZE)
return 0;
if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR)
return 0;
for (i = 0; i < err_list_len; i++)
if (error_list[i].value == err_id)
break;
/* if err_id is not in the error_list then ignore it */
if (i != err_list_len) {
dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text);
return err_id;
}
return 0;
}
static int atmel_ecc_wakeup(struct i2c_client *client)
{
struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
u8 status[STATUS_RSP_SIZE];
int ret;
/*
* The device ignores any levels or transitions on the SCL pin when the
* device is idle, asleep or during waking up. Don't check for error
* when waking up the device.
*/
i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz);
/*
* Wait to wake the device. Typical execution times for ecdh and genkey
* are around tens of milliseconds. Delta is chosen to 50 microseconds.
*/
usleep_range(TWHI_MIN, TWHI_MAX);
ret = i2c_master_recv(client, status, STATUS_SIZE);
if (ret < 0)
return ret;
return atmel_ecc_status(&client->dev, status);
}
static int atmel_ecc_sleep(struct i2c_client *client)
{
u8 sleep = SLEEP_TOKEN;
return i2c_master_send(client, &sleep, 1);
}
static void atmel_ecdh_done(struct atmel_ecc_work_data *work_data, void *areq,
static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq,
int status)
{
struct kpp_request *req = areq;
struct atmel_ecdh_ctx *ctx = work_data->ctx;
struct atmel_ecc_cmd *cmd = &work_data->cmd;
struct atmel_i2c_cmd *cmd = &work_data->cmd;
size_t copied, n_sz;
if (status)
@ -282,82 +73,6 @@ free_work_data:
kpp_request_complete(req, status);
}
/*
* atmel_ecc_send_receive() - send a command to the device and receive its
* response.
* @client: i2c client device
* @cmd : structure used to communicate with the device
*
* After the device receives a Wake token, a watchdog counter starts within the
* device. After the watchdog timer expires, the device enters sleep mode
* regardless of whether some I/O transmission or command execution is in
* progress. If a command is attempted when insufficient time remains prior to
* watchdog timer execution, the device will return the watchdog timeout error
* code without attempting to execute the command. There is no way to reset the
* counter other than to put the device into sleep or idle mode and then
* wake it up again.
*/
static int atmel_ecc_send_receive(struct i2c_client *client,
struct atmel_ecc_cmd *cmd)
{
struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
int ret;
mutex_lock(&i2c_priv->lock);
ret = atmel_ecc_wakeup(client);
if (ret)
goto err;
/* send the command */
ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE);
if (ret < 0)
goto err;
/* delay the appropriate amount of time for command to execute */
msleep(cmd->msecs);
/* receive the response */
ret = i2c_master_recv(client, cmd->data, cmd->rxsize);
if (ret < 0)
goto err;
/* put the device into low-power mode */
ret = atmel_ecc_sleep(client);
if (ret < 0)
goto err;
mutex_unlock(&i2c_priv->lock);
return atmel_ecc_status(&client->dev, cmd->data);
err:
mutex_unlock(&i2c_priv->lock);
return ret;
}
static void atmel_ecc_work_handler(struct work_struct *work)
{
struct atmel_ecc_work_data *work_data =
container_of(work, struct atmel_ecc_work_data, work);
struct atmel_ecc_cmd *cmd = &work_data->cmd;
struct i2c_client *client = work_data->ctx->client;
int status;
status = atmel_ecc_send_receive(client, cmd);
work_data->cbk(work_data, work_data->areq, status);
}
static void atmel_ecc_enqueue(struct atmel_ecc_work_data *work_data,
void (*cbk)(struct atmel_ecc_work_data *work_data,
void *areq, int status),
void *areq)
{
work_data->cbk = (void *)cbk;
work_data->areq = areq;
INIT_WORK(&work_data->work, atmel_ecc_work_handler);
schedule_work(&work_data->work);
}
static unsigned int atmel_ecdh_supported_curve(unsigned int curve_id)
{
if (curve_id == ECC_CURVE_NIST_P256)
@ -374,7 +89,7 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
unsigned int len)
{
struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
struct atmel_ecc_cmd *cmd;
struct atmel_i2c_cmd *cmd;
void *public_key;
struct ecdh params;
int ret = -ENOMEM;
@ -412,9 +127,9 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
ctx->do_fallback = false;
ctx->curve_id = params.curve_id;
atmel_ecc_init_genkey_cmd(cmd, DATA_SLOT_2);
atmel_i2c_init_genkey_cmd(cmd, DATA_SLOT_2);
ret = atmel_ecc_send_receive(ctx->client, cmd);
ret = atmel_i2c_send_receive(ctx->client, cmd);
if (ret)
goto free_public_key;
@ -444,6 +159,9 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req)
return crypto_kpp_generate_public_key(req);
}
if (!ctx->public_key)
return -EINVAL;
/* might want less than we've got */
nbytes = min_t(size_t, ATMEL_ECC_PUBKEY_SIZE, req->dst_len);
@ -461,7 +179,7 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
{
struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
struct atmel_ecc_work_data *work_data;
struct atmel_i2c_work_data *work_data;
gfp_t gfp;
int ret;
@ -482,12 +200,13 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
return -ENOMEM;
work_data->ctx = ctx;
work_data->client = ctx->client;
ret = atmel_ecc_init_ecdh_cmd(&work_data->cmd, req->src);
ret = atmel_i2c_init_ecdh_cmd(&work_data->cmd, req->src);
if (ret)
goto free_work_data;
atmel_ecc_enqueue(work_data, atmel_ecdh_done, req);
atmel_i2c_enqueue(work_data, atmel_ecdh_done, req);
return -EINPROGRESS;
@ -498,7 +217,7 @@ free_work_data:
static struct i2c_client *atmel_ecc_i2c_client_alloc(void)
{
struct atmel_ecc_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL;
struct atmel_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL;
struct i2c_client *client = ERR_PTR(-ENODEV);
int min_tfm_cnt = INT_MAX;
int tfm_cnt;
@ -533,7 +252,7 @@ static struct i2c_client *atmel_ecc_i2c_client_alloc(void)
static void atmel_ecc_i2c_client_free(struct i2c_client *client)
{
struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
atomic_dec(&i2c_priv->tfm_count);
}
@ -604,96 +323,18 @@ static struct kpp_alg atmel_ecdh = {
},
};
static inline size_t atmel_ecc_wake_token_sz(u32 bus_clk_rate)
{
u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC);
/* return the size of the wake_token in bytes */
return DIV_ROUND_UP(no_of_bits, 8);
}
static int device_sanity_check(struct i2c_client *client)
{
struct atmel_ecc_cmd *cmd;
int ret;
cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
if (!cmd)
return -ENOMEM;
atmel_ecc_init_read_cmd(cmd);
ret = atmel_ecc_send_receive(client, cmd);
if (ret)
goto free_cmd;
/*
* It is vital that the Configuration, Data and OTP zones be locked
* prior to release into the field of the system containing the device.
* Failure to lock these zones may permit modification of any secret
* keys and may lead to other security problems.
*/
if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) {
dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n");
ret = -ENOTSUPP;
}
/* fall through */
free_cmd:
kfree(cmd);
return ret;
}
static int atmel_ecc_probe(struct i2c_client *client,
const struct i2c_device_id *id)
{
struct atmel_ecc_i2c_client_priv *i2c_priv;
struct device *dev = &client->dev;
struct atmel_i2c_client_priv *i2c_priv;
int ret;
u32 bus_clk_rate;
if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
dev_err(dev, "I2C_FUNC_I2C not supported\n");
return -ENODEV;
}
ret = of_property_read_u32(client->adapter->dev.of_node,
"clock-frequency", &bus_clk_rate);
if (ret) {
dev_err(dev, "of: failed to read clock-frequency property\n");
return ret;
}
if (bus_clk_rate > 1000000L) {
dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n",
bus_clk_rate);
return -EINVAL;
}
i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL);
if (!i2c_priv)
return -ENOMEM;
i2c_priv->client = client;
mutex_init(&i2c_priv->lock);
/*
* WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate -
* 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz
* will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE.
*/
i2c_priv->wake_token_sz = atmel_ecc_wake_token_sz(bus_clk_rate);
memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token));
atomic_set(&i2c_priv->tfm_count, 0);
i2c_set_clientdata(client, i2c_priv);
ret = device_sanity_check(client);
ret = atmel_i2c_probe(client, id);
if (ret)
return ret;
i2c_priv = i2c_get_clientdata(client);
spin_lock(&driver_data.i2c_list_lock);
list_add_tail(&i2c_priv->i2c_client_list_node,
&driver_data.i2c_client_list);
@ -705,10 +346,10 @@ static int atmel_ecc_probe(struct i2c_client *client,
list_del(&i2c_priv->i2c_client_list_node);
spin_unlock(&driver_data.i2c_list_lock);
dev_err(dev, "%s alg registration failed\n",
dev_err(&client->dev, "%s alg registration failed\n",
atmel_ecdh.base.cra_driver_name);
} else {
dev_info(dev, "atmel ecc algorithms registered in /proc/crypto\n");
dev_info(&client->dev, "atmel ecc algorithms registered in /proc/crypto\n");
}
return ret;
@ -716,7 +357,7 @@ static int atmel_ecc_probe(struct i2c_client *client,
static int atmel_ecc_remove(struct i2c_client *client)
{
struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
/* Return EBUSY if i2c client already allocated. */
if (atomic_read(&i2c_priv->tfm_count)) {

View File

@ -1,116 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (c) 2017, Microchip Technology Inc.
* Author: Tudor Ambarus <tudor.ambarus@microchip.com>
*/
#ifndef __ATMEL_ECC_H__
#define __ATMEL_ECC_H__
#define ATMEL_ECC_PRIORITY 300
#define COMMAND 0x03 /* packet function */
#define SLEEP_TOKEN 0x01
#define WAKE_TOKEN_MAX_SIZE 8
/* Definitions of Data and Command sizes */
#define WORD_ADDR_SIZE 1
#define COUNT_SIZE 1
#define CRC_SIZE 2
#define CMD_OVERHEAD_SIZE (COUNT_SIZE + CRC_SIZE)
/* size in bytes of the n prime */
#define ATMEL_ECC_NIST_P256_N_SIZE 32
#define ATMEL_ECC_PUBKEY_SIZE (2 * ATMEL_ECC_NIST_P256_N_SIZE)
#define STATUS_RSP_SIZE 4
#define ECDH_RSP_SIZE (32 + CMD_OVERHEAD_SIZE)
#define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \
CMD_OVERHEAD_SIZE)
#define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE)
#define MAX_RSP_SIZE GENKEY_RSP_SIZE
/**
* atmel_ecc_cmd - structure used for communicating with the device.
* @word_addr: indicates the function of the packet sent to the device. This
* byte should have a value of COMMAND for normal operation.
* @count : number of bytes to be transferred to (or from) the device.
* @opcode : the command code.
* @param1 : the first parameter; always present.
* @param2 : the second parameter; always present.
* @data : optional remaining input data. Includes a 2-byte CRC.
* @rxsize : size of the data received from i2c client.
* @msecs : command execution time in milliseconds
*/
struct atmel_ecc_cmd {
u8 word_addr;
u8 count;
u8 opcode;
u8 param1;
u16 param2;
u8 data[MAX_RSP_SIZE];
u8 msecs;
u16 rxsize;
} __packed;
/* Status/Error codes */
#define STATUS_SIZE 0x04
#define STATUS_NOERR 0x00
#define STATUS_WAKE_SUCCESSFUL 0x11
static const struct {
u8 value;
const char *error_text;
} error_list[] = {
{ 0x01, "CheckMac or Verify miscompare" },
{ 0x03, "Parse Error" },
{ 0x05, "ECC Fault" },
{ 0x0F, "Execution Error" },
{ 0xEE, "Watchdog about to expire" },
{ 0xFF, "CRC or other communication error" },
};
/* Definitions for eeprom organization */
#define CONFIG_ZONE 0
/* Definitions for Indexes common to all commands */
#define RSP_DATA_IDX 1 /* buffer index of data in response */
#define DATA_SLOT_2 2 /* used for ECDH private key */
/* Definitions for the device lock state */
#define DEVICE_LOCK_ADDR 0x15
#define LOCK_VALUE_IDX (RSP_DATA_IDX + 2)
#define LOCK_CONFIG_IDX (RSP_DATA_IDX + 3)
/*
* Wake High delay to data communication (microseconds). SDA should be stable
* high for this entire duration.
*/
#define TWHI_MIN 1500
#define TWHI_MAX 1550
/* Wake Low duration */
#define TWLO_USEC 60
/* Command execution time (milliseconds) */
#define MAX_EXEC_TIME_ECDH 58
#define MAX_EXEC_TIME_GENKEY 115
#define MAX_EXEC_TIME_READ 1
/* Command opcode */
#define OPCODE_ECDH 0x43
#define OPCODE_GENKEY 0x40
#define OPCODE_READ 0x02
/* Definitions for the READ Command */
#define READ_COUNT 7
/* Definitions for the GenKey Command */
#define GENKEY_COUNT 7
#define GENKEY_MODE_PRIVATE 0x04
/* Definitions for the ECDH Command */
#define ECDH_COUNT 71
#define ECDH_PREFIX_MODE 0x00
#endif /* __ATMEL_ECC_H__ */

364
drivers/crypto/atmel-i2c.c Normal file
View File

@ -0,0 +1,364 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Microchip / Atmel ECC (I2C) driver.
*
* Copyright (c) 2017, Microchip Technology Inc.
* Author: Tudor Ambarus <tudor.ambarus@microchip.com>
*/
#include <linux/bitrev.h>
#include <linux/crc16.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/i2c.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include "atmel-i2c.h"
/**
* atmel_i2c_checksum() - Generate 16-bit CRC as required by ATMEL ECC.
* CRC16 verification of the count, opcode, param1, param2 and data bytes.
* The checksum is saved in little-endian format in the least significant
* two bytes of the command. CRC polynomial is 0x8005 and the initial register
* value should be zero.
*
* @cmd : structure used for communicating with the device.
*/
static void atmel_i2c_checksum(struct atmel_i2c_cmd *cmd)
{
u8 *data = &cmd->count;
size_t len = cmd->count - CRC_SIZE;
__le16 *__crc16 = (__le16 *)(data + len);
*__crc16 = cpu_to_le16(bitrev16(crc16(0, data, len)));
}
void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd)
{
cmd->word_addr = COMMAND;
cmd->opcode = OPCODE_READ;
/*
* Read the word from Configuration zone that contains the lock bytes
* (UserExtra, Selector, LockValue, LockConfig).
*/
cmd->param1 = CONFIG_ZONE;
cmd->param2 = cpu_to_le16(DEVICE_LOCK_ADDR);
cmd->count = READ_COUNT;
atmel_i2c_checksum(cmd);
cmd->msecs = MAX_EXEC_TIME_READ;
cmd->rxsize = READ_RSP_SIZE;
}
EXPORT_SYMBOL(atmel_i2c_init_read_cmd);
void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd)
{
cmd->word_addr = COMMAND;
cmd->opcode = OPCODE_RANDOM;
cmd->param1 = 0;
cmd->param2 = 0;
cmd->count = RANDOM_COUNT;
atmel_i2c_checksum(cmd);
cmd->msecs = MAX_EXEC_TIME_RANDOM;
cmd->rxsize = RANDOM_RSP_SIZE;
}
EXPORT_SYMBOL(atmel_i2c_init_random_cmd);
void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid)
{
cmd->word_addr = COMMAND;
cmd->count = GENKEY_COUNT;
cmd->opcode = OPCODE_GENKEY;
cmd->param1 = GENKEY_MODE_PRIVATE;
/* a random private key will be generated and stored in slot keyID */
cmd->param2 = cpu_to_le16(keyid);
atmel_i2c_checksum(cmd);
cmd->msecs = MAX_EXEC_TIME_GENKEY;
cmd->rxsize = GENKEY_RSP_SIZE;
}
EXPORT_SYMBOL(atmel_i2c_init_genkey_cmd);
int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd,
struct scatterlist *pubkey)
{
size_t copied;
cmd->word_addr = COMMAND;
cmd->count = ECDH_COUNT;
cmd->opcode = OPCODE_ECDH;
cmd->param1 = ECDH_PREFIX_MODE;
/* private key slot */
cmd->param2 = cpu_to_le16(DATA_SLOT_2);
/*
* The device only supports NIST P256 ECC keys. The public key size will
* always be the same. Use a macro for the key size to avoid unnecessary
* computations.
*/
copied = sg_copy_to_buffer(pubkey,
sg_nents_for_len(pubkey,
ATMEL_ECC_PUBKEY_SIZE),
cmd->data, ATMEL_ECC_PUBKEY_SIZE);
if (copied != ATMEL_ECC_PUBKEY_SIZE)
return -EINVAL;
atmel_i2c_checksum(cmd);
cmd->msecs = MAX_EXEC_TIME_ECDH;
cmd->rxsize = ECDH_RSP_SIZE;
return 0;
}
EXPORT_SYMBOL(atmel_i2c_init_ecdh_cmd);
/*
* After wake and after execution of a command, there will be error, status, or
* result bytes in the device's output register that can be retrieved by the
* system. When the length of that group is four bytes, the codes returned are
* detailed in error_list.
*/
static int atmel_i2c_status(struct device *dev, u8 *status)
{
size_t err_list_len = ARRAY_SIZE(error_list);
int i;
u8 err_id = status[1];
if (*status != STATUS_SIZE)
return 0;
if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR)
return 0;
for (i = 0; i < err_list_len; i++)
if (error_list[i].value == err_id)
break;
/* if err_id is not in the error_list then ignore it */
if (i != err_list_len) {
dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text);
return err_id;
}
return 0;
}
static int atmel_i2c_wakeup(struct i2c_client *client)
{
struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
u8 status[STATUS_RSP_SIZE];
int ret;
/*
* The device ignores any levels or transitions on the SCL pin when the
* device is idle, asleep or during waking up. Don't check for error
* when waking up the device.
*/
i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz);
/*
* Wait to wake the device. Typical execution times for ecdh and genkey
* are around tens of milliseconds. Delta is chosen to 50 microseconds.
*/
usleep_range(TWHI_MIN, TWHI_MAX);
ret = i2c_master_recv(client, status, STATUS_SIZE);
if (ret < 0)
return ret;
return atmel_i2c_status(&client->dev, status);
}
static int atmel_i2c_sleep(struct i2c_client *client)
{
u8 sleep = SLEEP_TOKEN;
return i2c_master_send(client, &sleep, 1);
}
/*
* atmel_i2c_send_receive() - send a command to the device and receive its
* response.
* @client: i2c client device
* @cmd : structure used to communicate with the device
*
* After the device receives a Wake token, a watchdog counter starts within the
* device. After the watchdog timer expires, the device enters sleep mode
* regardless of whether some I/O transmission or command execution is in
* progress. If a command is attempted when insufficient time remains prior to
* watchdog timer execution, the device will return the watchdog timeout error
* code without attempting to execute the command. There is no way to reset the
* counter other than to put the device into sleep or idle mode and then
* wake it up again.
*/
int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd)
{
struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
int ret;
mutex_lock(&i2c_priv->lock);
ret = atmel_i2c_wakeup(client);
if (ret)
goto err;
/* send the command */
ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE);
if (ret < 0)
goto err;
/* delay the appropriate amount of time for command to execute */
msleep(cmd->msecs);
/* receive the response */
ret = i2c_master_recv(client, cmd->data, cmd->rxsize);
if (ret < 0)
goto err;
/* put the device into low-power mode */
ret = atmel_i2c_sleep(client);
if (ret < 0)
goto err;
mutex_unlock(&i2c_priv->lock);
return atmel_i2c_status(&client->dev, cmd->data);
err:
mutex_unlock(&i2c_priv->lock);
return ret;
}
EXPORT_SYMBOL(atmel_i2c_send_receive);
static void atmel_i2c_work_handler(struct work_struct *work)
{
struct atmel_i2c_work_data *work_data =
container_of(work, struct atmel_i2c_work_data, work);
struct atmel_i2c_cmd *cmd = &work_data->cmd;
struct i2c_client *client = work_data->client;
int status;
status = atmel_i2c_send_receive(client, cmd);
work_data->cbk(work_data, work_data->areq, status);
}
void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data,
void (*cbk)(struct atmel_i2c_work_data *work_data,
void *areq, int status),
void *areq)
{
work_data->cbk = (void *)cbk;
work_data->areq = areq;
INIT_WORK(&work_data->work, atmel_i2c_work_handler);
schedule_work(&work_data->work);
}
EXPORT_SYMBOL(atmel_i2c_enqueue);
static inline size_t atmel_i2c_wake_token_sz(u32 bus_clk_rate)
{
u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC);
/* return the size of the wake_token in bytes */
return DIV_ROUND_UP(no_of_bits, 8);
}
static int device_sanity_check(struct i2c_client *client)
{
struct atmel_i2c_cmd *cmd;
int ret;
cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
if (!cmd)
return -ENOMEM;
atmel_i2c_init_read_cmd(cmd);
ret = atmel_i2c_send_receive(client, cmd);
if (ret)
goto free_cmd;
/*
* It is vital that the Configuration, Data and OTP zones be locked
* prior to release into the field of the system containing the device.
* Failure to lock these zones may permit modification of any secret
* keys and may lead to other security problems.
*/
if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) {
dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n");
ret = -ENOTSUPP;
}
/* fall through */
free_cmd:
kfree(cmd);
return ret;
}
int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id)
{
struct atmel_i2c_client_priv *i2c_priv;
struct device *dev = &client->dev;
int ret;
u32 bus_clk_rate;
if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
dev_err(dev, "I2C_FUNC_I2C not supported\n");
return -ENODEV;
}
bus_clk_rate = i2c_acpi_find_bus_speed(&client->adapter->dev);
if (!bus_clk_rate) {
ret = device_property_read_u32(&client->adapter->dev,
"clock-frequency", &bus_clk_rate);
if (ret) {
dev_err(dev, "failed to read clock-frequency property\n");
return ret;
}
}
if (bus_clk_rate > 1000000L) {
dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n",
bus_clk_rate);
return -EINVAL;
}
i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL);
if (!i2c_priv)
return -ENOMEM;
i2c_priv->client = client;
mutex_init(&i2c_priv->lock);
/*
* WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate -
* 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz
* will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE.
*/
i2c_priv->wake_token_sz = atmel_i2c_wake_token_sz(bus_clk_rate);
memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token));
atomic_set(&i2c_priv->tfm_count, 0);
i2c_set_clientdata(client, i2c_priv);
ret = device_sanity_check(client);
if (ret)
return ret;
return 0;
}
EXPORT_SYMBOL(atmel_i2c_probe);
MODULE_AUTHOR("Tudor Ambarus <tudor.ambarus@microchip.com>");
MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver");
MODULE_LICENSE("GPL v2");

197
drivers/crypto/atmel-i2c.h Normal file
View File

@ -0,0 +1,197 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (c) 2017, Microchip Technology Inc.
* Author: Tudor Ambarus <tudor.ambarus@microchip.com>
*/
#ifndef __ATMEL_I2C_H__
#define __ATMEL_I2C_H__
#include <linux/hw_random.h>
#include <linux/types.h>
#define ATMEL_ECC_PRIORITY 300
#define COMMAND 0x03 /* packet function */
#define SLEEP_TOKEN 0x01
#define WAKE_TOKEN_MAX_SIZE 8
/* Definitions of Data and Command sizes */
#define WORD_ADDR_SIZE 1
#define COUNT_SIZE 1
#define CRC_SIZE 2
#define CMD_OVERHEAD_SIZE (COUNT_SIZE + CRC_SIZE)
/* size in bytes of the n prime */
#define ATMEL_ECC_NIST_P256_N_SIZE 32
#define ATMEL_ECC_PUBKEY_SIZE (2 * ATMEL_ECC_NIST_P256_N_SIZE)
#define STATUS_RSP_SIZE 4
#define ECDH_RSP_SIZE (32 + CMD_OVERHEAD_SIZE)
#define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \
CMD_OVERHEAD_SIZE)
#define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE)
#define RANDOM_RSP_SIZE (32 + CMD_OVERHEAD_SIZE)
#define MAX_RSP_SIZE GENKEY_RSP_SIZE
/**
* atmel_i2c_cmd - structure used for communicating with the device.
* @word_addr: indicates the function of the packet sent to the device. This
* byte should have a value of COMMAND for normal operation.
* @count : number of bytes to be transferred to (or from) the device.
* @opcode : the command code.
* @param1 : the first parameter; always present.
* @param2 : the second parameter; always present.
* @data : optional remaining input data. Includes a 2-byte CRC.
* @rxsize : size of the data received from i2c client.
* @msecs : command execution time in milliseconds
*/
struct atmel_i2c_cmd {
u8 word_addr;
u8 count;
u8 opcode;
u8 param1;
__le16 param2;
u8 data[MAX_RSP_SIZE];
u8 msecs;
u16 rxsize;
} __packed;
/* Status/Error codes */
#define STATUS_SIZE 0x04
#define STATUS_NOERR 0x00
#define STATUS_WAKE_SUCCESSFUL 0x11
static const struct {
u8 value;
const char *error_text;
} error_list[] = {
{ 0x01, "CheckMac or Verify miscompare" },
{ 0x03, "Parse Error" },
{ 0x05, "ECC Fault" },
{ 0x0F, "Execution Error" },
{ 0xEE, "Watchdog about to expire" },
{ 0xFF, "CRC or other communication error" },
};
/* Definitions for eeprom organization */
#define CONFIG_ZONE 0
/* Definitions for Indexes common to all commands */
#define RSP_DATA_IDX 1 /* buffer index of data in response */
#define DATA_SLOT_2 2 /* used for ECDH private key */
/* Definitions for the device lock state */
#define DEVICE_LOCK_ADDR 0x15
#define LOCK_VALUE_IDX (RSP_DATA_IDX + 2)
#define LOCK_CONFIG_IDX (RSP_DATA_IDX + 3)
/*
* Wake High delay to data communication (microseconds). SDA should be stable
* high for this entire duration.
*/
#define TWHI_MIN 1500
#define TWHI_MAX 1550
/* Wake Low duration */
#define TWLO_USEC 60
/* Command execution time (milliseconds) */
#define MAX_EXEC_TIME_ECDH 58
#define MAX_EXEC_TIME_GENKEY 115
#define MAX_EXEC_TIME_READ 1
#define MAX_EXEC_TIME_RANDOM 50
/* Command opcode */
#define OPCODE_ECDH 0x43
#define OPCODE_GENKEY 0x40
#define OPCODE_READ 0x02
#define OPCODE_RANDOM 0x1b
/* Definitions for the READ Command */
#define READ_COUNT 7
/* Definitions for the RANDOM Command */
#define RANDOM_COUNT 7
/* Definitions for the GenKey Command */
#define GENKEY_COUNT 7
#define GENKEY_MODE_PRIVATE 0x04
/* Definitions for the ECDH Command */
#define ECDH_COUNT 71
#define ECDH_PREFIX_MODE 0x00
/* Used for binding tfm objects to i2c clients. */
struct atmel_ecc_driver_data {
struct list_head i2c_client_list;
spinlock_t i2c_list_lock;
} ____cacheline_aligned;
/**
* atmel_i2c_client_priv - i2c_client private data
* @client : pointer to i2c client device
* @i2c_client_list_node: part of i2c_client_list
* @lock : lock for sending i2c commands
* @wake_token : wake token array of zeros
* @wake_token_sz : size in bytes of the wake_token
* @tfm_count : number of active crypto transformations on i2c client
*
* Reads and writes from/to the i2c client are sequential. The first byte
* transmitted to the device is treated as the byte size. Any attempt to send
* more than this number of bytes will cause the device to not ACK those bytes.
* After the host writes a single command byte to the input buffer, reads are
* prohibited until after the device completes command execution. Use a mutex
* when sending i2c commands.
*/
struct atmel_i2c_client_priv {
struct i2c_client *client;
struct list_head i2c_client_list_node;
struct mutex lock;
u8 wake_token[WAKE_TOKEN_MAX_SIZE];
size_t wake_token_sz;
atomic_t tfm_count ____cacheline_aligned;
struct hwrng hwrng;
};
/**
* atmel_i2c_work_data - data structure representing the work
* @ctx : transformation context.
* @cbk : pointer to a callback function to be invoked upon completion of this
* request. This has the form:
* callback(struct atmel_i2c_work_data *work_data, void *areq, u8 status)
* where:
* @work_data: data structure representing the work
* @areq : optional pointer to an argument passed with the original
* request.
* @status : status returned from the i2c client device or i2c error.
* @areq: optional pointer to a user argument for use at callback time.
* @work: describes the task to be executed.
* @cmd : structure used for communicating with the device.
*/
struct atmel_i2c_work_data {
void *ctx;
struct i2c_client *client;
void (*cbk)(struct atmel_i2c_work_data *work_data, void *areq,
int status);
void *areq;
struct work_struct work;
struct atmel_i2c_cmd cmd;
};
int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id);
void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data,
void (*cbk)(struct atmel_i2c_work_data *work_data,
void *areq, int status),
void *areq);
int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd);
void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd);
void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd);
void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid);
int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd,
struct scatterlist *pubkey);
#endif /* __ATMEL_I2C_H__ */

View File

@ -0,0 +1,171 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Microchip / Atmel SHA204A (I2C) driver.
*
* Copyright (c) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
*/
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/i2c.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include "atmel-i2c.h"
static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data,
void *areq, int status)
{
struct atmel_i2c_client_priv *i2c_priv = work_data->ctx;
struct hwrng *rng = areq;
if (status)
dev_warn_ratelimited(&i2c_priv->client->dev,
"i2c transaction failed (%d)\n",
status);
rng->priv = (unsigned long)work_data;
atomic_dec(&i2c_priv->tfm_count);
}
static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data,
size_t max)
{
struct atmel_i2c_client_priv *i2c_priv;
struct atmel_i2c_work_data *work_data;
i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng);
/* keep maximum 1 asynchronous read in flight at any time */
if (!atomic_add_unless(&i2c_priv->tfm_count, 1, 1))
return 0;
if (rng->priv) {
work_data = (struct atmel_i2c_work_data *)rng->priv;
max = min(sizeof(work_data->cmd.data), max);
memcpy(data, &work_data->cmd.data, max);
rng->priv = 0;
} else {
work_data = kmalloc(sizeof(*work_data), GFP_ATOMIC);
if (!work_data)
return -ENOMEM;
work_data->ctx = i2c_priv;
work_data->client = i2c_priv->client;
max = 0;
}
atmel_i2c_init_random_cmd(&work_data->cmd);
atmel_i2c_enqueue(work_data, atmel_sha204a_rng_done, rng);
return max;
}
static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max,
bool wait)
{
struct atmel_i2c_client_priv *i2c_priv;
struct atmel_i2c_cmd cmd;
int ret;
if (!wait)
return atmel_sha204a_rng_read_nonblocking(rng, data, max);
i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng);
atmel_i2c_init_random_cmd(&cmd);
ret = atmel_i2c_send_receive(i2c_priv->client, &cmd);
if (ret)
return ret;
max = min(sizeof(cmd.data), max);
memcpy(data, cmd.data, max);
return max;
}
static int atmel_sha204a_probe(struct i2c_client *client,
const struct i2c_device_id *id)
{
struct atmel_i2c_client_priv *i2c_priv;
int ret;
ret = atmel_i2c_probe(client, id);
if (ret)
return ret;
i2c_priv = i2c_get_clientdata(client);
memset(&i2c_priv->hwrng, 0, sizeof(i2c_priv->hwrng));
i2c_priv->hwrng.name = dev_name(&client->dev);
i2c_priv->hwrng.read = atmel_sha204a_rng_read;
i2c_priv->hwrng.quality = 1024;
ret = hwrng_register(&i2c_priv->hwrng);
if (ret)
dev_warn(&client->dev, "failed to register RNG (%d)\n", ret);
return ret;
}
static int atmel_sha204a_remove(struct i2c_client *client)
{
struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
if (atomic_read(&i2c_priv->tfm_count)) {
dev_err(&client->dev, "Device is busy\n");
return -EBUSY;
}
if (i2c_priv->hwrng.priv)
kfree((void *)i2c_priv->hwrng.priv);
hwrng_unregister(&i2c_priv->hwrng);
return 0;
}
static const struct of_device_id atmel_sha204a_dt_ids[] = {
{ .compatible = "atmel,atsha204a", },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids);
static const struct i2c_device_id atmel_sha204a_id[] = {
{ "atsha204a", 0 },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(i2c, atmel_sha204a_id);
static struct i2c_driver atmel_sha204a_driver = {
.probe = atmel_sha204a_probe,
.remove = atmel_sha204a_remove,
.id_table = atmel_sha204a_id,
.driver.name = "atmel-sha204a",
.driver.of_match_table = of_match_ptr(atmel_sha204a_dt_ids),
};
static int __init atmel_sha204a_init(void)
{
return i2c_add_driver(&atmel_sha204a_driver);
}
static void __exit atmel_sha204a_exit(void)
{
flush_scheduled_work();
i2c_del_driver(&atmel_sha204a_driver);
}
module_init(atmel_sha204a_init);
module_exit(atmel_sha204a_exit);
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");

View File

@ -85,7 +85,7 @@ MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
* 0x70 - ring 2
* 0x78 - ring 3
*/
char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
/*
* Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN
* is set dynamically after reading SPU type from device tree.
@ -2083,7 +2083,7 @@ static int __ahash_init(struct ahash_request *req)
* Return: true if incremental hashing is not supported
* false otherwise
*/
bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
static bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
{
struct spu_hw *spu = &iproc_priv.spu;
@ -4809,7 +4809,7 @@ static int spu_dt_read(struct platform_device *pdev)
return 0;
}
int bcm_spu_probe(struct platform_device *pdev)
static int bcm_spu_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct spu_hw *spu = &iproc_priv.spu;
@ -4853,7 +4853,7 @@ failure:
return err;
}
int bcm_spu_remove(struct platform_device *pdev)
static int bcm_spu_remove(struct platform_device *pdev)
{
int i;
struct device *dev = &pdev->dev;

View File

@ -38,21 +38,21 @@ enum spu2_proto_sel {
SPU2_DTLS_AEAD = 10
};
char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256",
static char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256",
"DES", "3DES"
};
char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", "XTS",
"CCM", "GCM"
static char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB",
"XTS", "CCM", "GCM"
};
char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256",
static char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256",
"Reserved", "Reserved", "MD5", "SHA1", "SHA224", "SHA256", "SHA384",
"SHA512", "SHA512/224", "SHA512/256", "SHA3-224", "SHA3-256",
"SHA3-384", "SHA3-512"
};
char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC",
static char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC",
"Rabin", "CCM", "GCM", "Reserved"
};

View File

@ -2,6 +2,12 @@
config CRYPTO_DEV_FSL_CAAM_COMMON
tristate
config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
tristate
config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
tristate
config CRYPTO_DEV_FSL_CAAM
tristate "Freescale CAAM-Multicore platform driver backend"
depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE
@ -25,7 +31,7 @@ config CRYPTO_DEV_FSL_CAAM_DEBUG
Selecting this will enable printing of various debug
information in the CAAM driver.
config CRYPTO_DEV_FSL_CAAM_JR
menuconfig CRYPTO_DEV_FSL_CAAM_JR
tristate "Freescale CAAM Job Ring driver backend"
default y
help
@ -86,8 +92,9 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
threshold. Range is 1-65535.
config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
tristate "Register algorithm implementations with the Crypto API"
bool "Register algorithm implementations with the Crypto API"
default y
select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
select CRYPTO_AEAD
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
@ -97,13 +104,11 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
scatterlist crypto API (such as the linux native IPSec
stack) to the SEC4 via job ring.
To compile this as a module, choose M here: the module
will be called caamalg.
config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
tristate "Queue Interface as Crypto API backend"
bool "Queue Interface as Crypto API backend"
depends on FSL_DPAA && NET
default y
select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
help
@ -114,33 +119,26 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
assigned to the kernel should also be more than the number of
job rings.
To compile this as a module, choose M here: the module
will be called caamalg_qi.
config CRYPTO_DEV_FSL_CAAM_AHASH_API
tristate "Register hash algorithm implementations with Crypto API"
bool "Register hash algorithm implementations with Crypto API"
default y
select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
select CRYPTO_HASH
help
Selecting this will offload ahash for users of the
scatterlist crypto API to the SEC4 via job ring.
To compile this as a module, choose M here: the module
will be called caamhash.
config CRYPTO_DEV_FSL_CAAM_PKC_API
tristate "Register public key cryptography implementations with Crypto API"
bool "Register public key cryptography implementations with Crypto API"
default y
select CRYPTO_RSA
help
Selecting this will allow SEC Public key support for RSA.
Supported cryptographic primitives: encryption, decryption,
signature and verification.
To compile this as a module, choose M here: the module
will be called caam_pkc.
config CRYPTO_DEV_FSL_CAAM_RNG_API
tristate "Register caam device for hwrng API"
bool "Register caam device for hwrng API"
default y
select CRYPTO_RNG
select HW_RANDOM
@ -148,9 +146,6 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API
Selecting this will register the SEC4 hardware rng to
the hw_random API for suppying the kernel entropy pool.
To compile this as a module, choose M here: the module
will be called caamrng.
endif # CRYPTO_DEV_FSL_CAAM_JR
endif # CRYPTO_DEV_FSL_CAAM
@ -160,6 +155,8 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM
depends on FSL_MC_DPIO
depends on NETDEVICES
select CRYPTO_DEV_FSL_CAAM_COMMON
select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
select CRYPTO_BLKCIPHER
select CRYPTO_AUTHENC
select CRYPTO_AEAD
@ -171,12 +168,3 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM
To compile this as a module, choose M here: the module
will be called dpaa2_caam.
config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \
CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI || \
CRYPTO_DEV_FSL_DPAA2_CAAM)
config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
def_tristate (CRYPTO_DEV_FSL_CAAM_AHASH_API || \
CRYPTO_DEV_FSL_DPAA2_CAAM)

View File

@ -11,20 +11,20 @@ ccflags-y += -DVERSION=\"\"
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o
caam-objs := ctrl.o
caam_jr-objs := jr.o key_gen.o
caam_pkc-y := caampkc.o pkc_desc.o
caam-y := ctrl.o
caam_jr-y := jr.o key_gen.o
caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o
caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o
ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),)
ccflags-y += -DCONFIG_CAAM_QI
caam-objs += qi.o
endif
obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o

View File

@ -77,13 +77,6 @@
#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
#ifdef DEBUG
/* for print_hex_dumps with line references */
#define debug(format, arg...) printk(format, arg)
#else
#define debug(format, arg...)
#endif
struct caam_alg_entry {
int class1_alg_type;
int class2_alg_type;
@ -583,13 +576,11 @@ static int aead_setkey(struct crypto_aead *aead,
if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
goto badkey;
#ifdef DEBUG
printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
keys.authkeylen + keys.enckeylen, keys.enckeylen,
keys.authkeylen);
print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
/*
* If DKP is supported, use it in the shared descriptor to generate
@ -623,11 +614,10 @@ static int aead_setkey(struct crypto_aead *aead,
memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
keys.enckeylen, ctx->dir);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
ctx->adata.keylen_pad + keys.enckeylen, 1);
#endif
skip_split_key:
ctx->cdata.keylen = keys.enckeylen;
@ -678,10 +668,8 @@ static int gcm_setkey(struct crypto_aead *aead,
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
memcpy(ctx->key, key, keylen);
dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
@ -699,10 +687,8 @@ static int rfc4106_setkey(struct crypto_aead *aead,
if (keylen < 4)
return -EINVAL;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
memcpy(ctx->key, key, keylen);
@ -725,10 +711,8 @@ static int rfc4543_setkey(struct crypto_aead *aead,
if (keylen < 4)
return -EINVAL;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
memcpy(ctx->key, key, keylen);
@ -757,10 +741,8 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
OP_ALG_AAI_CTR_MOD128);
const bool is_rfc3686 = alg->caam.rfc3686;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
/*
* AES-CTR needs to load IV in CONTEXT1 reg
* at an offset of 128bits (16bytes)
@ -916,7 +898,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
}
if (iv_dma)
dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL);
if (sec4_sg_bytes)
dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
DMA_TO_DEVICE);
@ -949,9 +931,7 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
struct aead_request *req = context;
struct aead_edesc *edesc;
#ifdef DEBUG
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
@ -971,9 +951,7 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
struct aead_request *req = context;
struct aead_edesc *edesc;
#ifdef DEBUG
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
@ -1001,33 +979,32 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
int ivsize = crypto_skcipher_ivsize(skcipher);
#ifdef DEBUG
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents > 1 ? 100 : ivsize, 1);
#endif
caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
skcipher_unmap(jrdev, edesc, req);
/*
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block. This is used e.g. by the CTS mode.
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
if (ivsize)
scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
ivsize, ivsize, 0);
if (ivsize) {
memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
ivsize);
print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents > 1 ? 100 : ivsize, 1);
}
caam_dump_sg("dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
kfree(edesc);
@ -1039,26 +1016,35 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
{
struct skcipher_request *req = context;
struct skcipher_edesc *edesc;
#ifdef DEBUG
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
int ivsize = crypto_skcipher_ivsize(skcipher);
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
#endif
caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
skcipher_unmap(jrdev, edesc, req);
/*
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
if (ivsize) {
memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
ivsize);
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
ivsize, 1);
}
caam_dump_sg("dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
skcipher_unmap(jrdev, edesc, req);
kfree(edesc);
skcipher_request_complete(req, err);
@ -1106,6 +1092,7 @@ static void init_aead_job(struct aead_request *req,
if (unlikely(req->src != req->dst)) {
if (!edesc->mapped_dst_nents) {
dst_dma = 0;
out_options = 0;
} else if (edesc->mapped_dst_nents == 1) {
dst_dma = sg_dma_address(req->dst);
out_options = 0;
@ -1249,6 +1236,7 @@ static void init_skcipher_job(struct skcipher_request *req,
{
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct device *jrdev = ctx->jrdev;
int ivsize = crypto_skcipher_ivsize(skcipher);
u32 *desc = edesc->hw_desc;
u32 *sh_desc;
@ -1256,13 +1244,12 @@ static void init_skcipher_job(struct skcipher_request *req,
dma_addr_t src_dma, dst_dma, ptr;
int len, sec4_sg_index = 0;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
print_hex_dump_debug("presciv@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
pr_err("asked=%d, cryptlen%d\n",
dev_dbg(jrdev, "asked=%d, cryptlen%d\n",
(int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
#endif
caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ",
caam_dump_sg("src @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->src,
edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
@ -1285,7 +1272,7 @@ static void init_skcipher_job(struct skcipher_request *req,
if (likely(req->src == req->dst)) {
dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
out_options = in_options;
} else if (edesc->mapped_dst_nents == 1) {
} else if (!ivsize && edesc->mapped_dst_nents == 1) {
dst_dma = sg_dma_address(req->dst);
} else {
dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
@ -1293,7 +1280,7 @@ static void init_skcipher_job(struct skcipher_request *req,
out_options = LDST_SGF;
}
append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options);
append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options);
}
/*
@ -1309,37 +1296,36 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
int src_len, dst_len = 0;
struct aead_edesc *edesc;
int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
unsigned int authsize = ctx->authsize;
if (unlikely(req->dst != req->src)) {
src_nents = sg_nents_for_len(req->src, req->assoclen +
req->cryptlen);
src_len = req->assoclen + req->cryptlen;
dst_len = src_len + (encrypt ? authsize : (-authsize));
src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
req->assoclen + req->cryptlen);
src_len);
return ERR_PTR(src_nents);
}
dst_nents = sg_nents_for_len(req->dst, req->assoclen +
req->cryptlen +
(encrypt ? authsize :
(-authsize)));
dst_nents = sg_nents_for_len(req->dst, dst_len);
if (unlikely(dst_nents < 0)) {
dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
req->assoclen + req->cryptlen +
(encrypt ? authsize : (-authsize)));
dst_len);
return ERR_PTR(dst_nents);
}
} else {
src_nents = sg_nents_for_len(req->src, req->assoclen +
req->cryptlen +
(encrypt ? authsize : 0));
src_len = req->assoclen + req->cryptlen +
(encrypt ? authsize : 0);
src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
req->assoclen + req->cryptlen +
(encrypt ? authsize : 0));
src_len);
return ERR_PTR(src_nents);
}
}
@ -1380,8 +1366,16 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
}
}
/*
* HW reads 4 S/G entries at a time; make sure the reads don't go beyond
* the end of the table by allocating more S/G entries.
*/
sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
if (mapped_dst_nents > 1)
sec4_sg_len += pad_sg_nents(mapped_dst_nents);
else
sec4_sg_len = pad_sg_nents(sec4_sg_len);
sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
/* allocate space for base edesc and hw desc commands, link tables */
@ -1403,12 +1397,12 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
sec4_sg_index = 0;
if (mapped_src_nents > 1) {
sg_to_sec4_sg_last(req->src, mapped_src_nents,
sg_to_sec4_sg_last(req->src, src_len,
edesc->sec4_sg + sec4_sg_index, 0);
sec4_sg_index += mapped_src_nents;
}
if (mapped_dst_nents > 1) {
sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
sg_to_sec4_sg_last(req->dst, dst_len,
edesc->sec4_sg + sec4_sg_index, 0);
}
@ -1446,11 +1440,10 @@ static int gcm_encrypt(struct aead_request *req)
/* Create and submit job descriptor */
init_gcm_job(req, edesc, all_contig, true);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
@ -1556,11 +1549,10 @@ static int aead_encrypt(struct aead_request *req)
/* Create and submit job descriptor */
init_authenc_job(req, edesc, all_contig, true);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
@ -1591,11 +1583,10 @@ static int gcm_decrypt(struct aead_request *req)
/* Create and submit job descriptor*/
init_gcm_job(req, edesc, all_contig, false);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
@ -1627,7 +1618,7 @@ static int aead_decrypt(struct aead_request *req)
u32 *desc;
int ret = 0;
caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ",
caam_dump_sg("dec src@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->src,
req->assoclen + req->cryptlen, 1);
@ -1639,11 +1630,10 @@ static int aead_decrypt(struct aead_request *req)
/* Create and submit job descriptor*/
init_authenc_job(req, edesc, all_contig, false);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
@ -1719,7 +1709,29 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
else
sec4_sg_ents = mapped_src_nents + !!ivsize;
dst_sg_idx = sec4_sg_ents;
sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
/*
* Input, output HW S/G tables: [IV, src][dst, IV]
* IV entries point to the same buffer
* If src == dst, S/G entries are reused (S/G tables overlap)
*
* HW reads 4 S/G entries at a time; make sure the reads don't go beyond
* the end of the table by allocating more S/G entries. Logic:
* if (output S/G)
* pad output S/G, if needed
* else if (input S/G) ...
* pad input S/G, if needed
*/
if (ivsize || mapped_dst_nents > 1) {
if (req->src == req->dst)
sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents);
else
sec4_sg_ents += pad_sg_nents(mapped_dst_nents +
!!ivsize);
} else {
sec4_sg_ents = pad_sg_nents(sec4_sg_ents);
}
sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
/*
@ -1744,10 +1756,10 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
/* Make sure IV is located in a DMAable area */
if (ivsize) {
iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
memcpy(iv, req->iv, ivsize);
iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE);
iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
if (dma_mapping_error(jrdev, iv_dma)) {
dev_err(jrdev, "unable to map IV\n");
caam_unmap(jrdev, req->src, req->dst, src_nents,
@ -1759,13 +1771,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
}
if (dst_sg_idx)
sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg +
sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg +
!!ivsize, 0);
if (mapped_dst_nents > 1) {
sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
edesc->sec4_sg + dst_sg_idx, 0);
}
if (req->src != req->dst && (ivsize || mapped_dst_nents > 1))
sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg +
dst_sg_idx, 0);
if (ivsize)
dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx +
mapped_dst_nents, iv_dma, ivsize, 0);
if (ivsize || mapped_dst_nents > 1)
sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx +
mapped_dst_nents);
if (sec4_sg_bytes) {
edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
@ -1782,11 +1801,9 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
edesc->iv_dma = iv_dma;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "skcipher sec4_sg@" __stringify(__LINE__)": ",
print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
sec4_sg_bytes, 1);
#endif
return edesc;
}
@ -1807,11 +1824,11 @@ static int skcipher_encrypt(struct skcipher_request *req)
/* Create and submit job descriptor*/
init_skcipher_job(req, edesc, true);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, skcipher_encrypt_done, req);
@ -1830,7 +1847,6 @@ static int skcipher_decrypt(struct skcipher_request *req)
struct skcipher_edesc *edesc;
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
int ivsize = crypto_skcipher_ivsize(skcipher);
struct device *jrdev = ctx->jrdev;
u32 *desc;
int ret = 0;
@ -1840,22 +1856,13 @@ static int skcipher_decrypt(struct skcipher_request *req)
if (IS_ERR(edesc))
return PTR_ERR(edesc);
/*
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block.
*/
if (ivsize)
scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
ivsize, ivsize, 0);
/* Create and submit job descriptor*/
init_skcipher_job(req, edesc, false);
desc = edesc->hw_desc;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
ret = caam_jr_enqueue(jrdev, desc, skcipher_decrypt_done, req);
if (!ret) {
@ -3444,7 +3451,7 @@ static void caam_aead_exit(struct crypto_aead *tfm)
caam_exit_common(crypto_aead_ctx(tfm));
}
static void __exit caam_algapi_exit(void)
void caam_algapi_exit(void)
{
int i;
@ -3489,43 +3496,15 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
alg->exit = caam_aead_exit;
}
static int __init caam_algapi_init(void)
int caam_algapi_init(struct device *ctrldev)
{
struct device_node *dev_node;
struct platform_device *pdev;
struct caam_drv_private *priv;
struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
int i = 0, err = 0;
u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
u32 arc4_inst;
unsigned int md_limit = SHA512_DIGEST_SIZE;
bool registered = false, gcm_support;
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
if (!dev_node) {
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
if (!dev_node)
return -ENODEV;
}
pdev = of_find_device_by_node(dev_node);
if (!pdev) {
of_node_put(dev_node);
return -ENODEV;
}
priv = dev_get_drvdata(&pdev->dev);
of_node_put(dev_node);
/*
* If priv is NULL, it's probably because the caam driver wasn't
* properly initialized (e.g. RNG4 init failed). Thus, bail out here.
*/
if (!priv) {
err = -ENODEV;
goto out_put_dev;
}
/*
* Register crypto algorithms the device supports.
* First, detect presence and attributes of DES, AES, and MD blocks.
@ -3668,14 +3647,5 @@ static int __init caam_algapi_init(void)
if (registered)
pr_info("caam algorithms registered in /proc/crypto\n");
out_put_dev:
put_device(&pdev->dev);
return err;
}
module_init(caam_algapi_init);
module_exit(caam_algapi_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("FSL CAAM support for crypto API");
MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");

View File

@ -33,12 +33,11 @@ static inline void append_dec_op1(u32 *desc, u32 type)
}
jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
append_operation(desc, type | OP_ALG_AS_INITFINAL |
OP_ALG_DECRYPT);
append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
set_jump_tgt_here(desc, jump_cmd);
append_operation(desc, type | OP_ALG_AS_INITFINAL |
OP_ALG_DECRYPT | OP_ALG_AAI_DK);
append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
OP_ALG_AAI_DK);
set_jump_tgt_here(desc, uncond_jump_cmd);
}
@ -115,11 +114,9 @@ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"aead null enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
@ -204,11 +201,9 @@ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"aead null dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
@ -358,10 +353,9 @@ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
@ -475,10 +469,9 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
@ -613,11 +606,9 @@ copy_iv:
append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"aead givenc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
@ -742,10 +733,9 @@ void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
@ -838,10 +828,9 @@ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
@ -933,11 +922,9 @@ void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"rfc4106 enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
@ -1030,11 +1017,9 @@ void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"rfc4106 dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
@ -1115,11 +1100,9 @@ void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"rfc4543 enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
@ -1205,11 +1188,9 @@ void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"rfc4543 dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
@ -1410,17 +1391,21 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
LDST_OFFSET_SHIFT));
/* Load operation */
append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
OP_ALG_ENCRYPT);
/* Perform operation */
skcipher_append_src_dst(desc);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"skcipher enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
/* Store IV */
if (ivsize)
append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
LDST_CLASS_1_CCB | (ctx1_iv_off <<
LDST_OFFSET_SHIFT));
print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
@ -1479,7 +1464,7 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
/* Choose operation */
if (ctx1_iv_off)
append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
OP_ALG_DECRYPT);
else
append_dec_op1(desc, cdata->algtype);
@ -1487,11 +1472,15 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
/* Perform operation */
skcipher_append_src_dst(desc);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"skcipher dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
/* Store IV */
if (ivsize)
append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
LDST_CLASS_1_CCB | (ctx1_iv_off <<
LDST_OFFSET_SHIFT));
print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
}
EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
@ -1538,11 +1527,13 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
/* Perform operation */
skcipher_append_src_dst(desc);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"xts skcipher enc shdesc@" __stringify(__LINE__) ": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
/* Store upper 8B of IV */
append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
(0x20 << LDST_OFFSET_SHIFT));
print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
": ", DUMP_PREFIX_ADDRESS, 16, 4,
desc, desc_bytes(desc), 1);
}
EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
@ -1588,11 +1579,13 @@ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
/* Perform operation */
skcipher_append_src_dst(desc);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"xts skcipher dec shdesc@" __stringify(__LINE__) ": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
/* Store upper 8B of IV */
append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
(0x20 << LDST_OFFSET_SHIFT));
print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
}
EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);

View File

@ -44,9 +44,9 @@
#define DESC_SKCIPHER_BASE (3 * CAAM_CMD_SZ)
#define DESC_SKCIPHER_ENC_LEN (DESC_SKCIPHER_BASE + \
20 * CAAM_CMD_SZ)
21 * CAAM_CMD_SZ)
#define DESC_SKCIPHER_DEC_LEN (DESC_SKCIPHER_BASE + \
15 * CAAM_CMD_SZ)
16 * CAAM_CMD_SZ)
void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
unsigned int icvsize, int era);

View File

@ -4,7 +4,7 @@
* Based on caamalg.c
*
* Copyright 2013-2016 Freescale Semiconductor, Inc.
* Copyright 2016-2018 NXP
* Copyright 2016-2019 NXP
*/
#include "compat.h"
@ -214,13 +214,11 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
goto badkey;
#ifdef DEBUG
dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
keys.authkeylen + keys.enckeylen, keys.enckeylen,
keys.authkeylen);
print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
/*
* If DKP is supported, use it in the shared descriptor to generate
@ -237,7 +235,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
memcpy(ctx->key, keys.authkey, keys.authkeylen);
memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
keys.enckeylen);
dma_sync_single_for_device(jrdev, ctx->key_dma,
dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
ctx->adata.keylen_pad +
keys.enckeylen, ctx->dir);
goto skip_split_key;
@ -251,8 +249,9 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
/* postpend encryption key to auth split key */
memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
keys.enckeylen, ctx->dir);
dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
ctx->adata.keylen_pad + keys.enckeylen,
ctx->dir);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
@ -386,13 +385,12 @@ static int gcm_setkey(struct crypto_aead *aead,
struct device *jrdev = ctx->jrdev;
int ret;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
memcpy(ctx->key, key, keylen);
dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen,
ctx->dir);
ctx->cdata.keylen = keylen;
ret = gcm_set_sh_desc(aead);
@ -485,10 +483,8 @@ static int rfc4106_setkey(struct crypto_aead *aead,
if (keylen < 4)
return -EINVAL;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
memcpy(ctx->key, key, keylen);
/*
@ -496,8 +492,8 @@ static int rfc4106_setkey(struct crypto_aead *aead,
* in the nonce. Update the AES key length.
*/
ctx->cdata.keylen = keylen - 4;
dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
ctx->dir);
dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
ctx->cdata.keylen, ctx->dir);
ret = rfc4106_set_sh_desc(aead);
if (ret)
@ -589,10 +585,8 @@ static int rfc4543_setkey(struct crypto_aead *aead,
if (keylen < 4)
return -EINVAL;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
memcpy(ctx->key, key, keylen);
/*
@ -600,8 +594,8 @@ static int rfc4543_setkey(struct crypto_aead *aead,
* in the nonce. Update the AES key length.
*/
ctx->cdata.keylen = keylen - 4;
dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
ctx->dir);
dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
ctx->cdata.keylen, ctx->dir);
ret = rfc4543_set_sh_desc(aead);
if (ret)
@ -644,10 +638,9 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
const bool is_rfc3686 = alg->caam.rfc3686;
int ret = 0;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
/*
* AES-CTR needs to load IV in CONTEXT1 reg
* at an offset of 128bits (16bytes)
@ -838,7 +831,8 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
static void caam_unmap(struct device *dev, struct scatterlist *src,
struct scatterlist *dst, int src_nents,
int dst_nents, dma_addr_t iv_dma, int ivsize,
dma_addr_t qm_sg_dma, int qm_sg_bytes)
enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
int qm_sg_bytes)
{
if (dst != src) {
if (src_nents)
@ -850,7 +844,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
}
if (iv_dma)
dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
if (qm_sg_bytes)
dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
}
@ -863,7 +857,8 @@ static void aead_unmap(struct device *dev,
int ivsize = crypto_aead_ivsize(aead);
caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
edesc->qm_sg_bytes);
dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
}
@ -874,7 +869,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
int ivsize = crypto_skcipher_ivsize(skcipher);
caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
edesc->qm_sg_bytes);
}
static void aead_done(struct caam_drv_req *drv_req, u32 status)
@ -924,6 +920,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
int src_len, dst_len = 0;
struct aead_edesc *edesc;
dma_addr_t qm_sg_dma, iv_dma = 0;
int ivsize = 0;
@ -945,13 +942,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
}
if (likely(req->src == req->dst)) {
src_nents = sg_nents_for_len(req->src, req->assoclen +
req->cryptlen +
(encrypt ? authsize : 0));
src_len = req->assoclen + req->cryptlen +
(encrypt ? authsize : 0);
src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
req->assoclen + req->cryptlen +
(encrypt ? authsize : 0));
src_len);
qi_cache_free(edesc);
return ERR_PTR(src_nents);
}
@ -964,23 +961,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
return ERR_PTR(-ENOMEM);
}
} else {
src_nents = sg_nents_for_len(req->src, req->assoclen +
req->cryptlen);
src_len = req->assoclen + req->cryptlen;
dst_len = src_len + (encrypt ? authsize : (-authsize));
src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
req->assoclen + req->cryptlen);
src_len);
qi_cache_free(edesc);
return ERR_PTR(src_nents);
}
dst_nents = sg_nents_for_len(req->dst, req->assoclen +
req->cryptlen +
(encrypt ? authsize :
(-authsize)));
dst_nents = sg_nents_for_len(req->dst, dst_len);
if (unlikely(dst_nents < 0)) {
dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
req->assoclen + req->cryptlen +
(encrypt ? authsize : (-authsize)));
dst_len);
qi_cache_free(edesc);
return ERR_PTR(dst_nents);
}
@ -1019,9 +1014,24 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
/*
* Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
* Input is not contiguous.
* HW reads 4 S/G entries at a time; make sure the reads don't go beyond
* the end of the table by allocating more S/G entries. Logic:
* if (src != dst && output S/G)
* pad output S/G, if needed
* else if (src == dst && S/G)
* overlapping S/Gs; pad one of them
* else if (input S/G) ...
* pad input S/G, if needed
*/
qm_sg_ents = 1 + !!ivsize + mapped_src_nents +
(mapped_dst_nents > 1 ? mapped_dst_nents : 0);
qm_sg_ents = 1 + !!ivsize + mapped_src_nents;
if (mapped_dst_nents > 1)
qm_sg_ents += pad_sg_nents(mapped_dst_nents);
else if ((req->src == req->dst) && (mapped_src_nents > 1))
qm_sg_ents = max(pad_sg_nents(qm_sg_ents),
1 + !!ivsize + pad_sg_nents(mapped_src_nents));
else
qm_sg_ents = pad_sg_nents(qm_sg_ents);
sg_table = &edesc->sgt[0];
qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
@ -1029,7 +1039,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -1044,7 +1054,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (dma_mapping_error(qidev, iv_dma)) {
dev_err(qidev, "unable to map IV\n");
caam_unmap(qidev, req->src, req->dst, src_nents,
dst_nents, 0, 0, 0, 0);
dst_nents, 0, 0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -1063,7 +1073,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
dev_err(qidev, "unable to map assoclen\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
iv_dma, ivsize, 0, 0);
iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -1074,19 +1084,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
qm_sg_index++;
}
sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
qm_sg_index += mapped_src_nents;
if (mapped_dst_nents > 1)
sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
qm_sg_index, 0);
sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
if (dma_mapping_error(qidev, qm_sg_dma)) {
dev_err(qidev, "unable to map S/G table\n");
dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
iv_dma, ivsize, 0, 0);
iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -1109,7 +1118,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +
(1 + !!ivsize) * sizeof(*sg_table),
out_len, 0);
} else if (mapped_dst_nents == 1) {
} else if (mapped_dst_nents <= 1) {
dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len,
0);
} else {
@ -1182,33 +1191,28 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
struct device *qidev = caam_ctx->qidev;
int ivsize = crypto_skcipher_ivsize(skcipher);
#ifdef DEBUG
dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
#endif
dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
edesc = container_of(drv_req, typeof(*edesc), drv_req);
if (status)
caam_jr_strstatus(qidev, status);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ",
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents > 1 ? 100 : ivsize, 1);
caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
caam_dump_sg("dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
#endif
skcipher_unmap(qidev, edesc, req);
/*
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block. This is used e.g. by the CTS mode.
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
if (edesc->drv_req.drv_ctx->op_type == ENCRYPT)
scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
ivsize, ivsize, 0);
memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
qi_cache_free(edesc);
skcipher_request_complete(req, status);
@ -1276,14 +1280,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
qm_sg_ents = 1 + mapped_src_nents;
dst_sg_idx = qm_sg_ents;
qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
/*
* Input, output HW S/G tables: [IV, src][dst, IV]
* IV entries point to the same buffer
* If src == dst, S/G entries are reused (S/G tables overlap)
*
* HW reads 4 S/G entries at a time; make sure the reads don't go beyond
* the end of the table by allocating more S/G entries.
*/
if (req->src != req->dst)
qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
else
qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
ivsize > CAAM_QI_MEMCACHE_SIZE)) {
dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
@ -1292,7 +1308,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
if (unlikely(!edesc)) {
dev_err(qidev, "could not allocate extended descriptor\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
@ -1301,11 +1317,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
iv = (u8 *)(sg_table + qm_sg_ents);
memcpy(iv, req->iv, ivsize);
iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
if (dma_mapping_error(qidev, iv_dma)) {
dev_err(qidev, "unable to map IV\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -1319,18 +1335,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
edesc->drv_req.drv_ctx = drv_ctx;
dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
if (mapped_dst_nents > 1)
sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
dst_sg_idx, 0);
if (req->src != req->dst)
sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
ivsize, 0);
edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
DMA_TO_DEVICE);
if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
dev_err(qidev, "unable to map S/G table\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
iv_dma, ivsize, 0, 0);
iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -1340,16 +1358,14 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
ivsize + req->cryptlen, 0);
if (req->src == req->dst) {
if (req->src == req->dst)
dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
sizeof(*sg_table), req->cryptlen, 0);
} else if (mapped_dst_nents > 1) {
sizeof(*sg_table), req->cryptlen + ivsize,
0);
else
dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
sizeof(*sg_table), req->cryptlen, 0);
} else {
dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst),
req->cryptlen, 0);
}
sizeof(*sg_table), req->cryptlen + ivsize,
0);
return edesc;
}
@ -1359,7 +1375,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
struct skcipher_edesc *edesc;
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
int ivsize = crypto_skcipher_ivsize(skcipher);
int ret;
if (unlikely(caam_congested))
@ -1370,14 +1385,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
if (IS_ERR(edesc))
return PTR_ERR(edesc);
/*
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block.
*/
if (!encrypt)
scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
ivsize, ivsize, 0);
ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
if (!ret) {
ret = -EINPROGRESS;
@ -2382,6 +2389,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
bool uses_dkp)
{
struct caam_drv_private *priv;
struct device *dev;
/*
* distribute tfms across job rings to ensure in-order
@ -2393,16 +2401,17 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
return PTR_ERR(ctx->jrdev);
}
priv = dev_get_drvdata(ctx->jrdev->parent);
dev = ctx->jrdev->parent;
priv = dev_get_drvdata(dev);
if (priv->era >= 6 && uses_dkp)
ctx->dir = DMA_BIDIRECTIONAL;
else
ctx->dir = DMA_TO_DEVICE;
ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key),
ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key),
ctx->dir);
if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) {
dev_err(ctx->jrdev, "unable to map key\n");
if (dma_mapping_error(dev, ctx->key_dma)) {
dev_err(dev, "unable to map key\n");
caam_jr_free(ctx->jrdev);
return -ENOMEM;
}
@ -2411,7 +2420,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
ctx->qidev = priv->qidev;
ctx->qidev = dev;
spin_lock_init(&ctx->lock);
ctx->drv_ctx[ENCRYPT] = NULL;
@ -2445,7 +2454,8 @@ static void caam_exit_common(struct caam_ctx *ctx)
caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir);
dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key),
ctx->dir);
caam_jr_free(ctx->jrdev);
}
@ -2460,7 +2470,7 @@ static void caam_aead_exit(struct crypto_aead *tfm)
caam_exit_common(crypto_aead_ctx(tfm));
}
static void __exit caam_qi_algapi_exit(void)
void caam_qi_algapi_exit(void)
{
int i;
@ -2505,45 +2515,17 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
alg->exit = caam_aead_exit;
}
static int __init caam_qi_algapi_init(void)
int caam_qi_algapi_init(struct device *ctrldev)
{
struct device_node *dev_node;
struct platform_device *pdev;
struct device *ctrldev;
struct caam_drv_private *priv;
struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
int i = 0, err = 0;
u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;
unsigned int md_limit = SHA512_DIGEST_SIZE;
bool registered = false;
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
if (!dev_node) {
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
if (!dev_node)
return -ENODEV;
}
pdev = of_find_device_by_node(dev_node);
of_node_put(dev_node);
if (!pdev)
return -ENODEV;
ctrldev = &pdev->dev;
priv = dev_get_drvdata(ctrldev);
/*
* If priv is NULL, it's probably because the caam driver wasn't
* properly initialized (e.g. RNG4 init failed). Thus, bail out here.
*/
if (!priv || !priv->qi_present) {
err = -ENODEV;
goto out_put_dev;
}
if (caam_dpaa2) {
dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n");
err = -ENODEV;
goto out_put_dev;
return -ENODEV;
}
/*
@ -2598,7 +2580,7 @@ static int __init caam_qi_algapi_init(void)
err = crypto_register_skcipher(&t_alg->skcipher);
if (err) {
dev_warn(priv->qidev, "%s alg registration failed\n",
dev_warn(ctrldev, "%s alg registration failed\n",
t_alg->skcipher.base.cra_driver_name);
continue;
}
@ -2654,16 +2636,7 @@ static int __init caam_qi_algapi_init(void)
}
if (registered)
dev_info(priv->qidev, "algorithms registered in /proc/crypto\n");
dev_info(ctrldev, "algorithms registered in /proc/crypto\n");
out_put_dev:
put_device(ctrldev);
return err;
}
module_init(caam_qi_algapi_init);
module_exit(caam_qi_algapi_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend");
MODULE_AUTHOR("Freescale Semiconductor");

View File

@ -1,7 +1,7 @@
// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
/*
* Copyright 2015-2016 Freescale Semiconductor Inc.
* Copyright 2017-2018 NXP
* Copyright 2017-2019 NXP
*/
#include "compat.h"
@ -140,7 +140,8 @@ static struct caam_request *to_caam_req(struct crypto_async_request *areq)
static void caam_unmap(struct device *dev, struct scatterlist *src,
struct scatterlist *dst, int src_nents,
int dst_nents, dma_addr_t iv_dma, int ivsize,
dma_addr_t qm_sg_dma, int qm_sg_bytes)
enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
int qm_sg_bytes)
{
if (dst != src) {
if (src_nents)
@ -152,7 +153,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
}
if (iv_dma)
dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
if (qm_sg_bytes)
dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
@ -371,6 +372,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
int src_len, dst_len = 0;
struct aead_edesc *edesc;
dma_addr_t qm_sg_dma, iv_dma = 0;
int ivsize = 0;
@ -387,23 +389,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
}
if (unlikely(req->dst != req->src)) {
src_nents = sg_nents_for_len(req->src, req->assoclen +
req->cryptlen);
src_len = req->assoclen + req->cryptlen;
dst_len = src_len + (encrypt ? authsize : (-authsize));
src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
req->assoclen + req->cryptlen);
src_len);
qi_cache_free(edesc);
return ERR_PTR(src_nents);
}
dst_nents = sg_nents_for_len(req->dst, req->assoclen +
req->cryptlen +
(encrypt ? authsize :
(-authsize)));
dst_nents = sg_nents_for_len(req->dst, dst_len);
if (unlikely(dst_nents < 0)) {
dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
req->assoclen + req->cryptlen +
(encrypt ? authsize : (-authsize)));
dst_len);
qi_cache_free(edesc);
return ERR_PTR(dst_nents);
}
@ -434,13 +434,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
mapped_dst_nents = 0;
}
} else {
src_nents = sg_nents_for_len(req->src, req->assoclen +
req->cryptlen +
(encrypt ? authsize : 0));
src_len = req->assoclen + req->cryptlen +
(encrypt ? authsize : 0);
src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
req->assoclen + req->cryptlen +
(encrypt ? authsize : 0));
src_len);
qi_cache_free(edesc);
return ERR_PTR(src_nents);
}
@ -460,9 +460,25 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
/*
* Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
* Input is not contiguous.
* HW reads 4 S/G entries at a time; make sure the reads don't go beyond
* the end of the table by allocating more S/G entries. Logic:
* if (src != dst && output S/G)
* pad output S/G, if needed
* else if (src == dst && S/G)
* overlapping S/Gs; pad one of them
* else if (input S/G) ...
* pad input S/G, if needed
*/
qm_sg_nents = 1 + !!ivsize + mapped_src_nents +
(mapped_dst_nents > 1 ? mapped_dst_nents : 0);
qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
if (mapped_dst_nents > 1)
qm_sg_nents += pad_sg_nents(mapped_dst_nents);
else if ((req->src == req->dst) && (mapped_src_nents > 1))
qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
1 + !!ivsize +
pad_sg_nents(mapped_src_nents));
else
qm_sg_nents = pad_sg_nents(qm_sg_nents);
sg_table = &edesc->sgt[0];
qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
@ -470,7 +486,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_nents, ivsize);
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -485,7 +501,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (dma_mapping_error(dev, iv_dma)) {
dev_err(dev, "unable to map IV\n");
caam_unmap(dev, req->src, req->dst, src_nents,
dst_nents, 0, 0, 0, 0);
dst_nents, 0, 0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -509,7 +525,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (dma_mapping_error(dev, edesc->assoclen_dma)) {
dev_err(dev, "unable to map assoclen\n");
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
iv_dma, ivsize, 0, 0);
iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -520,19 +536,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
qm_sg_index++;
}
sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
qm_sg_index += mapped_src_nents;
if (mapped_dst_nents > 1)
sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
qm_sg_index, 0);
sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
if (dma_mapping_error(dev, qm_sg_dma)) {
dev_err(dev, "unable to map S/G table\n");
dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
iv_dma, ivsize, 0, 0);
iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -559,6 +574,14 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dpaa2_fl_set_addr(out_fle, qm_sg_dma +
(1 + !!ivsize) * sizeof(*sg_table));
}
} else if (!mapped_dst_nents) {
/*
* crypto engine requires the output entry to be present when
* "frame list" FD is used.
* Since engine does not support FMT=2'b11 (unused entry type),
* leaving out_fle zeroized is the best option.
*/
goto skip_out_fle;
} else if (mapped_dst_nents == 1) {
dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
@ -570,6 +593,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dpaa2_fl_set_len(out_fle, out_len);
skip_out_fle:
return edesc;
}
@ -1077,14 +1101,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
qm_sg_ents = 1 + mapped_src_nents;
dst_sg_idx = qm_sg_ents;
qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
/*
* Input, output HW S/G tables: [IV, src][dst, IV]
* IV entries point to the same buffer
* If src == dst, S/G entries are reused (S/G tables overlap)
*
* HW reads 4 S/G entries at a time; make sure the reads don't go beyond
* the end of the table by allocating more S/G entries.
*/
if (req->src != req->dst)
qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
else
qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
ivsize > CAAM_QI_MEMCACHE_SIZE)) {
dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
@ -1093,7 +1129,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
if (unlikely(!edesc)) {
dev_err(dev, "could not allocate extended descriptor\n");
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
@ -1102,11 +1138,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
iv = (u8 *)(sg_table + qm_sg_ents);
memcpy(iv, req->iv, ivsize);
iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
if (dma_mapping_error(dev, iv_dma)) {
dev_err(dev, "unable to map IV\n");
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
0, 0, 0);
0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -1117,18 +1153,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
edesc->qm_sg_bytes = qm_sg_bytes;
dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
if (mapped_dst_nents > 1)
sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
dst_sg_idx, 0);
if (req->src != req->dst)
sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
ivsize, 0);
edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
DMA_TO_DEVICE);
if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
dev_err(dev, "unable to map S/G table\n");
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
iv_dma, ivsize, 0, 0);
iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@ -1136,23 +1174,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
dpaa2_fl_set_final(in_fle, true);
dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
dpaa2_fl_set_len(out_fle, req->cryptlen);
dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
if (req->src == req->dst) {
dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
if (req->src == req->dst)
dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
sizeof(*sg_table));
} else if (mapped_dst_nents > 1) {
dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
else
dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
sizeof(*sg_table));
} else {
dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
}
return edesc;
}
@ -1164,7 +1198,8 @@ static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
int ivsize = crypto_aead_ivsize(aead);
caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
edesc->qm_sg_bytes);
dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
}
@ -1175,7 +1210,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
int ivsize = crypto_skcipher_ivsize(skcipher);
caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
edesc->qm_sg_bytes);
}
static void aead_encrypt_done(void *cbk_ctx, u32 status)
@ -1324,7 +1360,7 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents > 1 ? 100 : ivsize, 1);
caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ",
caam_dump_sg("dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
@ -1332,10 +1368,10 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
/*
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block. This is used e.g. by the CTS mode.
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize,
ivsize, 0);
memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
qi_cache_free(edesc);
skcipher_request_complete(req, ecode);
@ -1362,11 +1398,19 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents > 1 ? 100 : ivsize, 1);
caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ",
caam_dump_sg("dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
skcipher_unmap(ctx->dev, edesc, req);
/*
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
qi_cache_free(edesc);
skcipher_request_complete(req, ecode);
}
@ -1405,7 +1449,6 @@ static int skcipher_decrypt(struct skcipher_request *req)
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_request *caam_req = skcipher_request_ctx(req);
int ivsize = crypto_skcipher_ivsize(skcipher);
int ret;
/* allocate extended descriptor */
@ -1413,13 +1456,6 @@ static int skcipher_decrypt(struct skcipher_request *req)
if (IS_ERR(edesc))
return PTR_ERR(edesc);
/*
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block.
*/
scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize,
ivsize, 0);
caam_req->flc = &ctx->flc[DECRYPT];
caam_req->flc_dma = ctx->flc_dma[DECRYPT];
caam_req->cbk = skcipher_decrypt_done;
@ -3380,9 +3416,9 @@ static int ahash_update_ctx(struct ahash_request *req)
if (to_hash) {
struct dpaa2_sg_entry *sg_table;
int src_len = req->nbytes - *next_buflen;
src_nents = sg_nents_for_len(req->src,
req->nbytes - (*next_buflen));
src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(ctx->dev, "Invalid number of src SG.\n");
return src_nents;
@ -3409,7 +3445,7 @@ static int ahash_update_ctx(struct ahash_request *req)
edesc->src_nents = src_nents;
qm_sg_src_index = 1 + (*buflen ? 1 : 0);
qm_sg_bytes = (qm_sg_src_index + mapped_nents) *
qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
sizeof(*sg_table);
sg_table = &edesc->sgt[0];
@ -3423,7 +3459,7 @@ static int ahash_update_ctx(struct ahash_request *req)
goto unmap_ctx;
if (mapped_nents) {
sg_to_qm_sg_last(req->src, mapped_nents,
sg_to_qm_sg_last(req->src, src_len,
sg_table + qm_sg_src_index, 0);
if (*next_buflen)
scatterwalk_map_and_copy(next_buf, req->src,
@ -3494,7 +3530,7 @@ static int ahash_final_ctx(struct ahash_request *req)
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int buflen = *current_buflen(state);
int qm_sg_bytes, qm_sg_src_index;
int qm_sg_bytes;
int digestsize = crypto_ahash_digestsize(ahash);
struct ahash_edesc *edesc;
struct dpaa2_sg_entry *sg_table;
@ -3505,8 +3541,7 @@ static int ahash_final_ctx(struct ahash_request *req)
if (!edesc)
return -ENOMEM;
qm_sg_src_index = 1 + (buflen ? 1 : 0);
qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table);
qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
sg_table = &edesc->sgt[0];
ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
@ -3518,7 +3553,7 @@ static int ahash_final_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true);
dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
DMA_TO_DEVICE);
@ -3599,7 +3634,8 @@ static int ahash_finup_ctx(struct ahash_request *req)
edesc->src_nents = src_nents;
qm_sg_src_index = 1 + (buflen ? 1 : 0);
qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table);
qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
sizeof(*sg_table);
sg_table = &edesc->sgt[0];
ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
@ -3611,7 +3647,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0);
sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
DMA_TO_DEVICE);
@ -3696,8 +3732,8 @@ static int ahash_digest(struct ahash_request *req)
int qm_sg_bytes;
struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
qm_sg_bytes = mapped_nents * sizeof(*sg_table);
sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
qm_sg_bytes, DMA_TO_DEVICE);
if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
@ -3840,9 +3876,9 @@ static int ahash_update_no_ctx(struct ahash_request *req)
if (to_hash) {
struct dpaa2_sg_entry *sg_table;
int src_len = req->nbytes - *next_buflen;
src_nents = sg_nents_for_len(req->src,
req->nbytes - *next_buflen);
src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(ctx->dev, "Invalid number of src SG.\n");
return src_nents;
@ -3868,14 +3904,15 @@ static int ahash_update_no_ctx(struct ahash_request *req)
}
edesc->src_nents = src_nents;
qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table);
qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
sizeof(*sg_table);
sg_table = &edesc->sgt[0];
ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
if (ret)
goto unmap_ctx;
sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
if (*next_buflen)
scatterwalk_map_and_copy(next_buf, req->src,
@ -3987,14 +4024,14 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
}
edesc->src_nents = src_nents;
qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table);
qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
sg_table = &edesc->sgt[0];
ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
if (ret)
goto unmap;
sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
DMA_TO_DEVICE);
@ -4064,9 +4101,9 @@ static int ahash_update_first(struct ahash_request *req)
if (to_hash) {
struct dpaa2_sg_entry *sg_table;
int src_len = req->nbytes - *next_buflen;
src_nents = sg_nents_for_len(req->src,
req->nbytes - (*next_buflen));
src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(ctx->dev, "Invalid number of src SG.\n");
return src_nents;
@ -4101,8 +4138,9 @@ static int ahash_update_first(struct ahash_request *req)
if (mapped_nents > 1) {
int qm_sg_bytes;
sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
qm_sg_bytes = mapped_nents * sizeof(*sg_table);
sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
qm_sg_bytes = pad_sg_nents(mapped_nents) *
sizeof(*sg_table);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
qm_sg_bytes,
DMA_TO_DEVICE);

View File

@ -82,14 +82,6 @@
#define HASH_MSG_LEN 8
#define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
#ifdef DEBUG
/* for print_hex_dumps with line references */
#define debug(format, arg...) printk(format, arg)
#else
#define debug(format, arg...)
#endif
static struct list_head hash_list;
/* ahash per-session context */
@ -243,11 +235,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, true, ctrlpriv->era);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
desc_bytes(desc), ctx->dir);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"ahash update shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
/* ahash_update_first shared descriptor */
desc = ctx->sh_desc_update_first;
@ -255,11 +246,9 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, false, ctrlpriv->era);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
desc_bytes(desc), ctx->dir);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"ahash update first shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__)
": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
/* ahash_final shared descriptor */
desc = ctx->sh_desc_fin;
@ -267,11 +256,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, true, ctrlpriv->era);
dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
desc_bytes(desc), ctx->dir);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
/* ahash_digest shared descriptor */
desc = ctx->sh_desc_digest;
@ -279,12 +267,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, false, ctrlpriv->era);
dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
desc_bytes(desc), ctx->dir);
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"ahash digest shdesc@"__stringify(__LINE__)": ",
print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
return 0;
}
@ -328,9 +314,9 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, ctx->key_dma);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)" : ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)
" : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
/* shared descriptor for ahash_digest */
desc = ctx->sh_desc_digest;
@ -377,8 +363,8 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, 0);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)" : ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)
" : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
/* shared descriptor for ahash_digest */
@ -429,12 +415,11 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
print_hex_dump_debug("key_in@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
result.err = 0;
init_completion(&result.completion);
@ -444,11 +429,10 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
/* in progress */
wait_for_completion(&result.completion);
ret = result.err;
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"digested key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1);
#endif
print_hex_dump_debug("digested key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key,
digestsize, 1);
}
dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL);
@ -463,15 +447,14 @@ static int ahash_setkey(struct crypto_ahash *ahash,
const u8 *key, unsigned int keylen)
{
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct device *jrdev = ctx->jrdev;
int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
int digestsize = crypto_ahash_digestsize(ahash);
struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
int ret;
u8 *hashed_key = NULL;
#ifdef DEBUG
printk(KERN_ERR "keylen %d\n", keylen);
#endif
dev_dbg(jrdev, "keylen %d\n", keylen);
if (keylen > blocksize) {
hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
@ -600,11 +583,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
int digestsize = crypto_ahash_digestsize(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
#ifdef DEBUG
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
@ -614,11 +595,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
memcpy(req->result, state->caam_ctx, digestsize);
kfree(edesc);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
#endif
req->base.complete(&req->base, err);
}
@ -631,11 +610,9 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
#ifdef DEBUG
int digestsize = crypto_ahash_digestsize(ahash);
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
@ -645,15 +622,13 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
switch_buf(state);
kfree(edesc);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
print_hex_dump_debug("result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
req->base.complete(&req->base, err);
}
@ -666,11 +641,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
int digestsize = crypto_ahash_digestsize(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
#ifdef DEBUG
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
@ -680,11 +653,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
memcpy(req->result, state->caam_ctx, digestsize);
kfree(edesc);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
#endif
req->base.complete(&req->base, err);
}
@ -697,11 +668,9 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
#ifdef DEBUG
int digestsize = crypto_ahash_digestsize(ahash);
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
@ -711,15 +680,13 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
switch_buf(state);
kfree(edesc);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
print_hex_dump_debug("result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
req->base.complete(&req->base, err);
}
@ -759,9 +726,10 @@ static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
if (nents > 1 || first_sg) {
struct sec4_sg_entry *sg = edesc->sec4_sg;
unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
unsigned int sgsize = sizeof(*sg) *
pad_sg_nents(first_sg + nents);
sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0);
src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
if (dma_mapping_error(ctx->jrdev, src_dma)) {
@ -819,8 +787,10 @@ static int ahash_update_ctx(struct ahash_request *req)
}
if (to_hash) {
src_nents = sg_nents_for_len(req->src,
req->nbytes - (*next_buflen));
int pad_nents;
int src_len = req->nbytes - *next_buflen;
src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(jrdev, "Invalid number of src SG.\n");
return src_nents;
@ -838,15 +808,14 @@ static int ahash_update_ctx(struct ahash_request *req)
}
sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
sizeof(struct sec4_sg_entry);
pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents);
sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
/*
* allocate space for base edesc and hw desc commands,
* link tables
*/
edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
ctx->sh_desc_update,
edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update,
ctx->sh_desc_update_dma, flags);
if (!edesc) {
dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
@ -866,7 +835,7 @@ static int ahash_update_ctx(struct ahash_request *req)
goto unmap_ctx;
if (mapped_nents)
sg_to_sec4_sg_last(req->src, mapped_nents,
sg_to_sec4_sg_last(req->src, src_len,
edesc->sec4_sg + sec4_sg_src_index,
0);
else
@ -893,11 +862,9 @@ static int ahash_update_ctx(struct ahash_request *req)
append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
if (ret)
@ -910,13 +877,12 @@ static int ahash_update_ctx(struct ahash_request *req)
*buflen = *next_buflen;
*next_buflen = last_buflen;
}
#ifdef DEBUG
print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
print_hex_dump_debug("buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
*next_buflen, 1);
#endif
return ret;
unmap_ctx:
@ -935,18 +901,17 @@ static int ahash_final_ctx(struct ahash_request *req)
GFP_KERNEL : GFP_ATOMIC;
int buflen = *current_buflen(state);
u32 *desc;
int sec4_sg_bytes, sec4_sg_src_index;
int sec4_sg_bytes;
int digestsize = crypto_ahash_digestsize(ahash);
struct ahash_edesc *edesc;
int ret;
sec4_sg_src_index = 1 + (buflen ? 1 : 0);
sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) *
sizeof(struct sec4_sg_entry);
/* allocate space for base edesc and hw desc commands, link tables */
edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
flags);
edesc = ahash_edesc_alloc(ctx, 4, ctx->sh_desc_fin,
ctx->sh_desc_fin_dma, flags);
if (!edesc)
return -ENOMEM;
@ -963,7 +928,7 @@ static int ahash_final_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1);
sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0));
edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
sec4_sg_bytes, DMA_TO_DEVICE);
@ -977,10 +942,9 @@ static int ahash_final_ctx(struct ahash_request *req)
LDST_SGF);
append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
if (ret)
@ -1058,10 +1022,9 @@ static int ahash_finup_ctx(struct ahash_request *req)
append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
if (ret)
@ -1135,10 +1098,9 @@ static int ahash_digest(struct ahash_request *req)
return -ENOMEM;
}
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
if (!ret) {
@ -1190,10 +1152,9 @@ static int ahash_final_no_ctx(struct ahash_request *req)
if (ret)
goto unmap;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
if (!ret) {
@ -1246,8 +1207,10 @@ static int ahash_update_no_ctx(struct ahash_request *req)
}
if (to_hash) {
src_nents = sg_nents_for_len(req->src,
req->nbytes - *next_buflen);
int pad_nents;
int src_len = req->nbytes - *next_buflen;
src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(jrdev, "Invalid number of src SG.\n");
return src_nents;
@ -1264,14 +1227,14 @@ static int ahash_update_no_ctx(struct ahash_request *req)
mapped_nents = 0;
}
sec4_sg_bytes = (1 + mapped_nents) *
sizeof(struct sec4_sg_entry);
pad_nents = pad_sg_nents(1 + mapped_nents);
sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
/*
* allocate space for base edesc and hw desc commands,
* link tables
*/
edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
edesc = ahash_edesc_alloc(ctx, pad_nents,
ctx->sh_desc_update_first,
ctx->sh_desc_update_first_dma,
flags);
@ -1287,8 +1250,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
sg_to_sec4_sg_last(req->src, mapped_nents,
edesc->sec4_sg + 1, 0);
sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0);
if (*next_buflen) {
scatterwalk_map_and_copy(next_buf, req->src,
@ -1313,11 +1275,9 @@ static int ahash_update_no_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
if (ret)
@ -1333,13 +1293,12 @@ static int ahash_update_no_ctx(struct ahash_request *req)
*buflen = *next_buflen;
*next_buflen = 0;
}
#ifdef DEBUG
print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
print_hex_dump_debug("buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
*next_buflen, 1);
#endif
print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
1);
return ret;
unmap_ctx:
@ -1414,10 +1373,9 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
if (ret)
goto unmap;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
if (!ret) {
@ -1517,11 +1475,9 @@ static int ahash_update_first(struct ahash_request *req)
if (ret)
goto unmap_ctx;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
if (ret)
@ -1539,11 +1495,10 @@ static int ahash_update_first(struct ahash_request *req)
req->nbytes, 0);
switch_buf(state);
}
#ifdef DEBUG
print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
*next_buflen, 1);
#endif
print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
1);
return ret;
unmap_ctx:
@ -1930,7 +1885,7 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
caam_jr_free(ctx->jrdev);
}
static void __exit caam_algapi_hash_exit(void)
void caam_algapi_hash_exit(void)
{
struct caam_hash_alg *t_alg, *n;
@ -1988,40 +1943,13 @@ caam_hash_alloc(struct caam_hash_template *template,
return t_alg;
}
static int __init caam_algapi_hash_init(void)
int caam_algapi_hash_init(struct device *ctrldev)
{
struct device_node *dev_node;
struct platform_device *pdev;
int i = 0, err = 0;
struct caam_drv_private *priv;
struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
unsigned int md_limit = SHA512_DIGEST_SIZE;
u32 md_inst, md_vid;
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
if (!dev_node) {
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
if (!dev_node)
return -ENODEV;
}
pdev = of_find_device_by_node(dev_node);
if (!pdev) {
of_node_put(dev_node);
return -ENODEV;
}
priv = dev_get_drvdata(&pdev->dev);
of_node_put(dev_node);
/*
* If priv is NULL, it's probably because the caam driver wasn't
* properly initialized (e.g. RNG4 init failed). Thus, bail out here.
*/
if (!priv) {
err = -ENODEV;
goto out_put_dev;
}
/*
* Register crypto algorithms the device supports. First, identify
* presence and attributes of MD block.
@ -2042,10 +1970,8 @@ static int __init caam_algapi_hash_init(void)
* Skip registration of any hashing algorithms if MD block
* is not present.
*/
if (!md_inst) {
err = -ENODEV;
goto out_put_dev;
}
if (!md_inst)
return -ENODEV;
/* Limit digest size based on LP256 */
if (md_vid == CHA_VER_VID_MD_LP256)
@ -2102,14 +2028,5 @@ static int __init caam_algapi_hash_init(void)
list_add_tail(&t_alg->entry, &hash_list);
}
out_put_dev:
put_device(&pdev->dev);
return err;
}
module_init(caam_algapi_hash_init);
module_exit(caam_algapi_hash_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
MODULE_AUTHOR("Freescale Semiconductor - NMG");

View File

@ -3,7 +3,7 @@
* caam - Freescale FSL CAAM support for Public Key Cryptography
*
* Copyright 2016 Freescale Semiconductor, Inc.
* Copyright 2018 NXP
* Copyright 2018-2019 NXP
*
* There is no Shared Descriptor for PKC so that the Job Descriptor must carry
* all the desired key parameters, input and output pointers.
@ -24,12 +24,18 @@
sizeof(struct rsa_priv_f2_pdb))
#define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \
sizeof(struct rsa_priv_f3_pdb))
#define CAAM_RSA_MAX_INPUT_SIZE 512 /* for a 4096-bit modulus */
/* buffer filled with zeros, used for padding */
static u8 *zero_buffer;
static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc,
struct akcipher_request *req)
{
struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE);
dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
dma_unmap_sg(dev, req_ctx->fixup_src, edesc->src_nents, DMA_TO_DEVICE);
if (edesc->sec4_sg_bytes)
dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes,
@ -168,6 +174,13 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
akcipher_request_complete(req, err);
}
/**
* Count leading zeros, need it to strip, from a given scatterlist
*
* @sgl : scatterlist to count zeros from
* @nbytes: number of zeros, in bytes, to strip
* @flags : operation flags
*/
static int caam_rsa_count_leading_zeros(struct scatterlist *sgl,
unsigned int nbytes,
unsigned int flags)
@ -187,7 +200,8 @@ static int caam_rsa_count_leading_zeros(struct scatterlist *sgl,
lzeros = 0;
len = 0;
while (nbytes > 0) {
while (len && !*buff) {
/* do not strip more than given bytes */
while (len && !*buff && lzeros < nbytes) {
lzeros++;
len--;
buff++;
@ -218,6 +232,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
struct device *dev = ctx->dev;
struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
struct caam_rsa_key *key = &ctx->key;
struct rsa_edesc *edesc;
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
@ -225,22 +240,45 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
int sgc;
int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
int src_nents, dst_nents;
unsigned int diff_size = 0;
int lzeros;
lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len, sg_flags);
if (req->src_len > key->n_sz) {
/*
* strip leading zeros and
* return the number of zeros to skip
*/
lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len -
key->n_sz, sg_flags);
if (lzeros < 0)
return ERR_PTR(lzeros);
req->src_len -= lzeros;
req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros);
req_ctx->fixup_src = scatterwalk_ffwd(req_ctx->src, req->src,
lzeros);
req_ctx->fixup_src_len = req->src_len - lzeros;
} else {
/*
* input src is less then n key modulus,
* so there will be zero padding
*/
diff_size = key->n_sz - req->src_len;
req_ctx->fixup_src = req->src;
req_ctx->fixup_src_len = req->src_len;
}
src_nents = sg_nents_for_len(req->src, req->src_len);
src_nents = sg_nents_for_len(req_ctx->fixup_src,
req_ctx->fixup_src_len);
dst_nents = sg_nents_for_len(req->dst, req->dst_len);
if (src_nents > 1)
sec4_sg_len = src_nents;
if (!diff_size && src_nents == 1)
sec4_sg_len = 0; /* no need for an input hw s/g table */
else
sec4_sg_len = src_nents + !!diff_size;
sec4_sg_index = sec4_sg_len;
if (dst_nents > 1)
sec4_sg_len += dst_nents;
sec4_sg_len += pad_sg_nents(dst_nents);
else
sec4_sg_len = pad_sg_nents(sec4_sg_len);
sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
@ -250,7 +288,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
if (!edesc)
return ERR_PTR(-ENOMEM);
sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
sgc = dma_map_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE);
if (unlikely(!sgc)) {
dev_err(dev, "unable to map source\n");
goto src_fail;
@ -263,14 +301,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
}
edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen;
if (diff_size)
dma_to_sec4_sg_one(edesc->sec4_sg, ctx->padding_dma, diff_size,
0);
if (sec4_sg_index)
sg_to_sec4_sg_last(req_ctx->fixup_src, req_ctx->fixup_src_len,
edesc->sec4_sg + !!diff_size, 0);
sec4_sg_index = 0;
if (src_nents > 1) {
sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
sec4_sg_index += src_nents;
}
if (dst_nents > 1)
sg_to_sec4_sg_last(req->dst, dst_nents,
sg_to_sec4_sg_last(req->dst, req->dst_len,
edesc->sec4_sg + sec4_sg_index, 0);
/* Save nents for later use in Job Descriptor */
@ -289,12 +329,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
edesc->sec4_sg_bytes = sec4_sg_bytes;
print_hex_dump_debug("caampkc sec4_sg@" __stringify(__LINE__) ": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
edesc->sec4_sg_bytes, 1);
return edesc;
sec4_sg_fail:
dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
dst_fail:
dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
dma_unmap_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE);
src_fail:
kfree(edesc);
return ERR_PTR(-ENOMEM);
@ -304,6 +348,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req,
struct rsa_edesc *edesc)
{
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
struct caam_rsa_key *key = &ctx->key;
struct device *dev = ctx->dev;
@ -328,7 +373,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req,
pdb->f_dma = edesc->sec4_sg_dma;
sec4_sg_index += edesc->src_nents;
} else {
pdb->f_dma = sg_dma_address(req->src);
pdb->f_dma = sg_dma_address(req_ctx->fixup_src);
}
if (edesc->dst_nents > 1) {
@ -340,7 +385,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req,
}
pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz;
pdb->f_len = req->src_len;
pdb->f_len = req_ctx->fixup_src_len;
return 0;
}
@ -373,7 +418,9 @@ static int set_rsa_priv_f1_pdb(struct akcipher_request *req,
pdb->g_dma = edesc->sec4_sg_dma;
sec4_sg_index += edesc->src_nents;
} else {
pdb->g_dma = sg_dma_address(req->src);
struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
}
if (edesc->dst_nents > 1) {
@ -436,7 +483,9 @@ static int set_rsa_priv_f2_pdb(struct akcipher_request *req,
pdb->g_dma = edesc->sec4_sg_dma;
sec4_sg_index += edesc->src_nents;
} else {
pdb->g_dma = sg_dma_address(req->src);
struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
}
if (edesc->dst_nents > 1) {
@ -523,7 +572,9 @@ static int set_rsa_priv_f3_pdb(struct akcipher_request *req,
pdb->g_dma = edesc->sec4_sg_dma;
sec4_sg_index += edesc->src_nents;
} else {
pdb->g_dma = sg_dma_address(req->src);
struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
}
if (edesc->dst_nents > 1) {
@ -978,6 +1029,15 @@ static int caam_rsa_init_tfm(struct crypto_akcipher *tfm)
return PTR_ERR(ctx->dev);
}
ctx->padding_dma = dma_map_single(ctx->dev, zero_buffer,
CAAM_RSA_MAX_INPUT_SIZE - 1,
DMA_TO_DEVICE);
if (dma_mapping_error(ctx->dev, ctx->padding_dma)) {
dev_err(ctx->dev, "unable to map padding\n");
caam_jr_free(ctx->dev);
return -ENOMEM;
}
return 0;
}
@ -987,6 +1047,8 @@ static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm)
struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
struct caam_rsa_key *key = &ctx->key;
dma_unmap_single(ctx->dev, ctx->padding_dma, CAAM_RSA_MAX_INPUT_SIZE -
1, DMA_TO_DEVICE);
caam_rsa_free_key(key);
caam_jr_free(ctx->dev);
}
@ -1010,41 +1072,12 @@ static struct akcipher_alg caam_rsa = {
};
/* Public Key Cryptography module initialization handler */
static int __init caam_pkc_init(void)
int caam_pkc_init(struct device *ctrldev)
{
struct device_node *dev_node;
struct platform_device *pdev;
struct device *ctrldev;
struct caam_drv_private *priv;
struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
u32 pk_inst;
int err;
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
if (!dev_node) {
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
if (!dev_node)
return -ENODEV;
}
pdev = of_find_device_by_node(dev_node);
if (!pdev) {
of_node_put(dev_node);
return -ENODEV;
}
ctrldev = &pdev->dev;
priv = dev_get_drvdata(ctrldev);
of_node_put(dev_node);
/*
* If priv is NULL, it's probably because the caam driver wasn't
* properly initialized (e.g. RNG4 init failed). Thus, bail out here.
*/
if (!priv) {
err = -ENODEV;
goto out_put_dev;
}
/* Determine public key hardware accelerator presence. */
if (priv->era < 10)
pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
@ -1053,31 +1086,29 @@ static int __init caam_pkc_init(void)
pk_inst = rd_reg32(&priv->ctrl->vreg.pkha) & CHA_VER_NUM_MASK;
/* Do not register algorithms if PKHA is not present. */
if (!pk_inst) {
err = -ENODEV;
goto out_put_dev;
}
if (!pk_inst)
return 0;
/* allocate zero buffer, used for padding input */
zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_DMA |
GFP_KERNEL);
if (!zero_buffer)
return -ENOMEM;
err = crypto_register_akcipher(&caam_rsa);
if (err)
if (err) {
kfree(zero_buffer);
dev_warn(ctrldev, "%s alg registration failed\n",
caam_rsa.base.cra_driver_name);
else
} else {
dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n");
}
out_put_dev:
put_device(ctrldev);
return err;
}
static void __exit caam_pkc_exit(void)
void caam_pkc_exit(void)
{
kfree(zero_buffer);
crypto_unregister_akcipher(&caam_rsa);
}
module_init(caam_pkc_init);
module_exit(caam_pkc_exit);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API");
MODULE_AUTHOR("Freescale Semiconductor");

View File

@ -89,18 +89,25 @@ struct caam_rsa_key {
* caam_rsa_ctx - per session context.
* @key : RSA key in DMA zone
* @dev : device structure
* @padding_dma : dma address of padding, for adding it to the input
*/
struct caam_rsa_ctx {
struct caam_rsa_key key;
struct device *dev;
dma_addr_t padding_dma;
};
/**
* caam_rsa_req_ctx - per request context.
* @src : input scatterlist (stripped of leading zeros)
* @fixup_src : input scatterlist (that might be stripped of leading zeros)
* @fixup_src_len : length of the fixup_src input scatterlist
*/
struct caam_rsa_req_ctx {
struct scatterlist src[2];
struct scatterlist *fixup_src;
unsigned int fixup_src_len;
};
/**

View File

@ -3,7 +3,7 @@
* caam - Freescale FSL CAAM support for hw_random
*
* Copyright 2011 Freescale Semiconductor, Inc.
* Copyright 2018 NXP
* Copyright 2018-2019 NXP
*
* Based on caamalg.c crypto API driver.
*
@ -113,10 +113,8 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context)
/* Buffer refilled, invalidate cache */
dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, DMA_FROM_DEVICE);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "rng refreshed buf@: ",
DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1);
#endif
print_hex_dump_debug("rng refreshed buf@: ", DUMP_PREFIX_ADDRESS, 16, 4,
bd->buf, RN_BUF_SIZE, 1);
}
static inline int submit_job(struct caam_rng_ctx *ctx, int to_current)
@ -209,10 +207,10 @@ static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx)
dev_err(jrdev, "unable to map shared descriptor\n");
return -ENOMEM;
}
#ifdef DEBUG
print_hex_dump(KERN_ERR, "rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
print_hex_dump_debug("rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
desc, desc_bytes(desc), 1);
#endif
return 0;
}
@ -233,10 +231,10 @@ static inline int rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id)
}
append_seq_out_ptr_intlen(desc, bd->addr, RN_BUF_SIZE, 0);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
print_hex_dump_debug("rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
desc, desc_bytes(desc), 1);
#endif
return 0;
}
@ -296,47 +294,20 @@ static struct hwrng caam_rng = {
.read = caam_read,
};
static void __exit caam_rng_exit(void)
void caam_rng_exit(void)
{
caam_jr_free(rng_ctx->jrdev);
hwrng_unregister(&caam_rng);
kfree(rng_ctx);
}
static int __init caam_rng_init(void)
int caam_rng_init(struct device *ctrldev)
{
struct device *dev;
struct device_node *dev_node;
struct platform_device *pdev;
struct caam_drv_private *priv;
u32 rng_inst;
struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
int err;
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
if (!dev_node) {
dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
if (!dev_node)
return -ENODEV;
}
pdev = of_find_device_by_node(dev_node);
if (!pdev) {
of_node_put(dev_node);
return -ENODEV;
}
priv = dev_get_drvdata(&pdev->dev);
of_node_put(dev_node);
/*
* If priv is NULL, it's probably because the caam driver wasn't
* properly initialized (e.g. RNG4 init failed). Thus, bail out here.
*/
if (!priv) {
err = -ENODEV;
goto out_put_dev;
}
/* Check for an instantiated RNG before registration */
if (priv->era < 10)
rng_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
@ -344,16 +315,13 @@ static int __init caam_rng_init(void)
else
rng_inst = rd_reg32(&priv->ctrl->vreg.rng) & CHA_VER_NUM_MASK;
if (!rng_inst) {
err = -ENODEV;
goto out_put_dev;
}
if (!rng_inst)
return 0;
dev = caam_jr_alloc();
if (IS_ERR(dev)) {
pr_err("Job Ring Device allocation for transform failed\n");
err = PTR_ERR(dev);
goto out_put_dev;
return PTR_ERR(dev);
}
rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA | GFP_KERNEL);
if (!rng_ctx) {
@ -364,7 +332,6 @@ static int __init caam_rng_init(void)
if (err)
goto free_rng_ctx;
put_device(&pdev->dev);
dev_info(dev, "registering rng-caam\n");
return hwrng_register(&caam_rng);
@ -372,14 +339,5 @@ free_rng_ctx:
kfree(rng_ctx);
free_caam_alloc:
caam_jr_free(dev);
out_put_dev:
put_device(&pdev->dev);
return err;
}
module_init(caam_rng_init);
module_exit(caam_rng_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("FSL CAAM support for hw_random API");
MODULE_AUTHOR("Freescale Semiconductor - NMG");

View File

@ -3,7 +3,7 @@
* Controller-level driver, kernel property detection, initialization
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
* Copyright 2018 NXP
* Copyright 2018-2019 NXP
*/
#include <linux/device.h>
@ -323,8 +323,8 @@ static int caam_remove(struct platform_device *pdev)
of_platform_depopulate(ctrldev);
#ifdef CONFIG_CAAM_QI
if (ctrlpriv->qidev)
caam_qi_shutdown(ctrlpriv->qidev);
if (ctrlpriv->qi_init)
caam_qi_shutdown(ctrldev);
#endif
/*
@ -540,7 +540,8 @@ static int caam_probe(struct platform_device *pdev)
ctrlpriv->caam_ipg = clk;
if (!of_machine_is_compatible("fsl,imx7d") &&
!of_machine_is_compatible("fsl,imx7s")) {
!of_machine_is_compatible("fsl,imx7s") &&
!of_machine_is_compatible("fsl,imx7ulp")) {
clk = caam_drv_identify_clk(&pdev->dev, "mem");
if (IS_ERR(clk)) {
ret = PTR_ERR(clk);
@ -562,7 +563,8 @@ static int caam_probe(struct platform_device *pdev)
if (!of_machine_is_compatible("fsl,imx6ul") &&
!of_machine_is_compatible("fsl,imx7d") &&
!of_machine_is_compatible("fsl,imx7s")) {
!of_machine_is_compatible("fsl,imx7s") &&
!of_machine_is_compatible("fsl,imx7ulp")) {
clk = caam_drv_identify_clk(&pdev->dev, "emi_slow");
if (IS_ERR(clk)) {
ret = PTR_ERR(clk);
@ -702,12 +704,7 @@ static int caam_probe(struct platform_device *pdev)
}
ctrlpriv->era = caam_get_era(ctrl);
ret = of_platform_populate(nprop, caam_match, NULL, dev);
if (ret) {
dev_err(dev, "JR platform devices creation error\n");
goto iounmap_ctrl;
}
ctrlpriv->domain = iommu_get_domain_for_dev(dev);
#ifdef CONFIG_DEBUG_FS
/*
@ -721,19 +718,6 @@ static int caam_probe(struct platform_device *pdev)
ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root);
#endif
ring = 0;
for_each_available_child_of_node(nprop, np)
if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
((__force uint8_t *)ctrl +
(ring + JR_BLOCK_NUMBER) *
BLOCK_OFFSET
);
ctrlpriv->total_jobrs++;
ring++;
}
/* Check to see if (DPAA 1.x) QI present. If so, enable */
ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
if (ctrlpriv->qi_present && !caam_dpaa2) {
@ -752,6 +736,25 @@ static int caam_probe(struct platform_device *pdev)
#endif
}
ret = of_platform_populate(nprop, caam_match, NULL, dev);
if (ret) {
dev_err(dev, "JR platform devices creation error\n");
goto shutdown_qi;
}
ring = 0;
for_each_available_child_of_node(nprop, np)
if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
((__force uint8_t *)ctrl +
(ring + JR_BLOCK_NUMBER) *
BLOCK_OFFSET
);
ctrlpriv->total_jobrs++;
ring++;
}
/* If no QI and no rings specified, quit and go home */
if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) {
dev_err(dev, "no queues configured, terminating\n");
@ -898,6 +901,11 @@ caam_remove:
caam_remove(pdev);
return ret;
shutdown_qi:
#ifdef CONFIG_CAAM_QI
if (ctrlpriv->qi_init)
caam_qi_shutdown(dev);
#endif
iounmap_ctrl:
iounmap(ctrl);
disable_caam_emi_slow:

View File

@ -3,6 +3,7 @@
* caam descriptor construction helper functions
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
* Copyright 2019 NXP
*/
#ifndef DESC_CONSTR_H
@ -37,6 +38,16 @@
extern bool caam_little_end;
/*
* HW fetches 4 S/G table entries at a time, irrespective of how many entries
* are in the table. It's SW's responsibility to make sure these accesses
* do not have side effects.
*/
static inline int pad_sg_nents(int sg_nents)
{
return ALIGN(sg_nents, 4);
}
static inline int desc_len(u32 * const desc)
{
return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK;

View File

@ -13,7 +13,7 @@
#ifdef DEBUG
#include <linux/highmem.h>
void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
void caam_dump_sg(const char *prefix_str, int prefix_type,
int rowsize, int groupsize, struct scatterlist *sg,
size_t tlen, bool ascii)
{
@ -35,7 +35,7 @@ void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
buf = it_page + it->offset;
len = min_t(size_t, tlen, it->length);
print_hex_dump(level, prefix_str, prefix_type, rowsize,
print_hex_dump_debug(prefix_str, prefix_type, rowsize,
groupsize, buf, len, ascii);
tlen -= len;
@ -43,7 +43,7 @@ void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
}
}
#else
void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
void caam_dump_sg(const char *prefix_str, int prefix_type,
int rowsize, int groupsize, struct scatterlist *sg,
size_t tlen, bool ascii)
{}

View File

@ -17,7 +17,7 @@ void caam_strstatus(struct device *dev, u32 status, bool qi_v2);
#define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false)
#define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true)
void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
void caam_dump_sg(const char *prefix_str, int prefix_type,
int rowsize, int groupsize, struct scatterlist *sg,
size_t tlen, bool ascii);

View File

@ -4,7 +4,7 @@
* Private/internal definitions between modules
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
*
* Copyright 2019 NXP
*/
#ifndef INTERN_H
@ -63,10 +63,6 @@ struct caam_drv_private_jr {
* Driver-private storage for a single CAAM block instance
*/
struct caam_drv_private {
#ifdef CONFIG_CAAM_QI
struct device *qidev;
#endif
/* Physical-presence section */
struct caam_ctrl __iomem *ctrl; /* controller region */
struct caam_deco __iomem *deco; /* DECO/CCB views */
@ -74,12 +70,17 @@ struct caam_drv_private {
struct caam_queue_if __iomem *qi; /* QI control region */
struct caam_job_ring __iomem *jr[4]; /* JobR's register space */
struct iommu_domain *domain;
/*
* Detected geometry block. Filled in from device tree if powerpc,
* or from register-based version detection code
*/
u8 total_jobrs; /* Total Job Rings in device */
u8 qi_present; /* Nonzero if QI present in device */
#ifdef CONFIG_CAAM_QI
u8 qi_init; /* Nonzero if QI has been initialized */
#endif
u8 mc_en; /* Nonzero if MC f/w is active */
int secvio_irq; /* Security violation interrupt number */
int virt_en; /* Virtualization enabled in CAAM */
@ -107,8 +108,95 @@ struct caam_drv_private {
#endif
};
void caam_jr_algapi_init(struct device *dev);
void caam_jr_algapi_remove(struct device *dev);
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API
int caam_algapi_init(struct device *dev);
void caam_algapi_exit(void);
#else
static inline int caam_algapi_init(struct device *dev)
{
return 0;
}
static inline void caam_algapi_exit(void)
{
}
#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API */
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API
int caam_algapi_hash_init(struct device *dev);
void caam_algapi_hash_exit(void);
#else
static inline int caam_algapi_hash_init(struct device *dev)
{
return 0;
}
static inline void caam_algapi_hash_exit(void)
{
}
#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API */
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API
int caam_pkc_init(struct device *dev);
void caam_pkc_exit(void);
#else
static inline int caam_pkc_init(struct device *dev)
{
return 0;
}
static inline void caam_pkc_exit(void)
{
}
#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API */
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API
int caam_rng_init(struct device *dev);
void caam_rng_exit(void);
#else
static inline int caam_rng_init(struct device *dev)
{
return 0;
}
static inline void caam_rng_exit(void)
{
}
#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API */
#ifdef CONFIG_CAAM_QI
int caam_qi_algapi_init(struct device *dev);
void caam_qi_algapi_exit(void);
#else
static inline int caam_qi_algapi_init(struct device *dev)
{
return 0;
}
static inline void caam_qi_algapi_exit(void)
{
}
#endif /* CONFIG_CAAM_QI */
#ifdef CONFIG_DEBUG_FS
static int caam_debugfs_u64_get(void *data, u64 *val)

View File

@ -4,6 +4,7 @@
* JobR backend functionality
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
* Copyright 2019 NXP
*/
#include <linux/of_irq.h>
@ -23,6 +24,43 @@ struct jr_driver_data {
} ____cacheline_aligned;
static struct jr_driver_data driver_data;
static DEFINE_MUTEX(algs_lock);
static unsigned int active_devs;
static void register_algs(struct device *dev)
{
mutex_lock(&algs_lock);
if (++active_devs != 1)
goto algs_unlock;
caam_algapi_init(dev);
caam_algapi_hash_init(dev);
caam_pkc_init(dev);
caam_rng_init(dev);
caam_qi_algapi_init(dev);
algs_unlock:
mutex_unlock(&algs_lock);
}
static void unregister_algs(void)
{
mutex_lock(&algs_lock);
if (--active_devs != 0)
goto algs_unlock;
caam_qi_algapi_exit();
caam_rng_exit();
caam_pkc_exit();
caam_algapi_hash_exit();
caam_algapi_exit();
algs_unlock:
mutex_unlock(&algs_lock);
}
static int caam_reset_hw_jr(struct device *dev)
{
@ -109,6 +147,9 @@ static int caam_jr_remove(struct platform_device *pdev)
return -EBUSY;
}
/* Unregister JR-based RNG & crypto algorithms */
unregister_algs();
/* Remove the node from Physical JobR list maintained by driver */
spin_lock(&driver_data.jr_alloc_lock);
list_del(&jrpriv->list_node);
@ -541,6 +582,8 @@ static int caam_jr_probe(struct platform_device *pdev)
atomic_set(&jrpriv->tfm_count, 0);
register_algs(jrdev->parent);
return 0;
}

View File

@ -16,9 +16,7 @@ void split_key_done(struct device *dev, u32 *desc, u32 err,
{
struct split_key_result *res = context;
#ifdef DEBUG
dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
dev_dbg(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
if (err)
caam_jr_strstatus(dev, err);
@ -55,12 +53,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
adata->keylen_pad = split_key_pad_len(adata->algtype &
OP_ALG_ALGSEL_MASK);
#ifdef DEBUG
dev_err(jrdev, "split keylen %d split keylen padded %d\n",
dev_dbg(jrdev, "split keylen %d split keylen padded %d\n",
adata->keylen, adata->keylen_pad);
print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
#endif
if (adata->keylen_pad > max_keylen)
return -EINVAL;
@ -102,10 +98,9 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
append_fifo_store(desc, dma_addr, adata->keylen,
LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1);
result.err = 0;
init_completion(&result.completion);
@ -115,11 +110,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
/* in progress */
wait_for_completion(&result.completion);
ret = result.err;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_out,
adata->keylen_pad, 1);
#endif
}
dma_unmap_single(jrdev, dma_addr, adata->keylen_pad, DMA_BIDIRECTIONAL);

View File

@ -4,7 +4,7 @@
* Queue Interface backend functionality
*
* Copyright 2013-2016 Freescale Semiconductor, Inc.
* Copyright 2016-2017 NXP
* Copyright 2016-2017, 2019 NXP
*/
#include <linux/cpumask.h>
@ -18,6 +18,7 @@
#include "desc_constr.h"
#define PREHDR_RSLS_SHIFT 31
#define PREHDR_ABS BIT(25)
/*
* Use a reasonable backlog of frames (per CPU) as congestion threshold,
@ -58,11 +59,9 @@ static DEFINE_PER_CPU(int, last_cpu);
/*
* caam_qi_priv - CAAM QI backend private params
* @cgr: QMan congestion group
* @qi_pdev: platform device for QI backend
*/
struct caam_qi_priv {
struct qman_cgr cgr;
struct platform_device *qi_pdev;
};
static struct caam_qi_priv qipriv ____cacheline_aligned;
@ -95,6 +94,16 @@ static u64 times_congested;
*/
static struct kmem_cache *qi_cache;
static void *caam_iova_to_virt(struct iommu_domain *domain,
dma_addr_t iova_addr)
{
phys_addr_t phys_addr;
phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
return phys_to_virt(phys_addr);
}
int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
{
struct qm_fd fd;
@ -135,6 +144,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
const struct qm_fd *fd;
struct caam_drv_req *drv_req;
struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
struct caam_drv_private *priv = dev_get_drvdata(qidev);
fd = &msg->ern.fd;
@ -143,7 +153,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
return;
}
drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
if (!drv_req) {
dev_err(qidev,
"Can't find original request for CAAM response\n");
@ -346,6 +356,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
*/
drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
num_words);
drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
dma_sync_single_for_device(qidev, drv_ctx->context_a,
sizeof(drv_ctx->sh_desc) +
@ -401,6 +412,7 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
*/
drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
num_words);
drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc);
hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size,
@ -488,7 +500,7 @@ EXPORT_SYMBOL(caam_drv_ctx_rel);
void caam_qi_shutdown(struct device *qidev)
{
int i;
struct caam_qi_priv *priv = dev_get_drvdata(qidev);
struct caam_qi_priv *priv = &qipriv;
const cpumask_t *cpus = qman_affine_cpus();
for_each_cpu(i, cpus) {
@ -506,8 +518,6 @@ void caam_qi_shutdown(struct device *qidev)
qman_release_cgrid(priv->cgr.cgrid);
kmem_cache_destroy(qi_cache);
platform_device_unregister(priv->qi_pdev);
}
static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
@ -550,6 +560,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
struct caam_drv_req *drv_req;
const struct qm_fd *fd;
struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
struct caam_drv_private *priv = dev_get_drvdata(qidev);
u32 status;
if (caam_qi_napi_schedule(p, caam_napi))
@ -572,7 +583,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
return qman_cb_dqrr_consume;
}
drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
if (unlikely(!drv_req)) {
dev_err(qidev,
"Can't find original request for caam response\n");
@ -692,33 +703,17 @@ static void free_rsp_fqs(void)
int caam_qi_init(struct platform_device *caam_pdev)
{
int err, i;
struct platform_device *qi_pdev;
struct device *ctrldev = &caam_pdev->dev, *qidev;
struct caam_drv_private *ctrlpriv;
const cpumask_t *cpus = qman_affine_cpus();
static struct platform_device_info qi_pdev_info = {
.name = "caam_qi",
.id = PLATFORM_DEVID_NONE
};
qi_pdev_info.parent = ctrldev;
qi_pdev_info.dma_mask = dma_get_mask(ctrldev);
qi_pdev = platform_device_register_full(&qi_pdev_info);
if (IS_ERR(qi_pdev))
return PTR_ERR(qi_pdev);
set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev));
ctrlpriv = dev_get_drvdata(ctrldev);
qidev = &qi_pdev->dev;
qipriv.qi_pdev = qi_pdev;
dev_set_drvdata(qidev, &qipriv);
qidev = ctrldev;
/* Initialize the congestion detection */
err = init_cgr(qidev);
if (err) {
dev_err(qidev, "CGR initialization failed: %d\n", err);
platform_device_unregister(qi_pdev);
return err;
}
@ -727,7 +722,6 @@ int caam_qi_init(struct platform_device *caam_pdev)
if (err) {
dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
free_rsp_fqs();
platform_device_unregister(qi_pdev);
return err;
}
@ -750,15 +744,11 @@ int caam_qi_init(struct platform_device *caam_pdev)
napi_enable(irqtask);
}
/* Hook up QI device to parent controlling caam device */
ctrlpriv->qidev = qidev;
qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
SLAB_CACHE_DMA, NULL);
if (!qi_cache) {
dev_err(qidev, "Can't allocate CAAM cache\n");
free_rsp_fqs();
platform_device_unregister(qi_pdev);
return -ENOMEM;
}
@ -766,6 +756,8 @@ int caam_qi_init(struct platform_device *caam_pdev)
debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
&times_congested, &caam_fops_u64_ro);
#endif
ctrlpriv->qi_init = 1;
dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
return 0;
}

View File

@ -54,15 +54,19 @@ static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr,
* but does not have final bit; instead, returns last entry
*/
static inline struct qm_sg_entry *
sg_to_qm_sg(struct scatterlist *sg, int sg_count,
sg_to_qm_sg(struct scatterlist *sg, int len,
struct qm_sg_entry *qm_sg_ptr, u16 offset)
{
while (sg_count && sg) {
dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg),
sg_dma_len(sg), offset);
int ent_len;
while (len) {
ent_len = min_t(int, sg_dma_len(sg), len);
dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len,
offset);
qm_sg_ptr++;
sg = sg_next(sg);
sg_count--;
len -= ent_len;
}
return qm_sg_ptr - 1;
}
@ -71,10 +75,10 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count,
* convert scatterlist to h/w link table format
* scatterlist must have been previously dma mapped
*/
static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count,
static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len,
struct qm_sg_entry *qm_sg_ptr, u16 offset)
{
qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset);
qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset);
qm_sg_entry_set_f(qm_sg_ptr, qm_sg_entry_get_len(qm_sg_ptr));
}

View File

@ -25,15 +25,19 @@ static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr,
* but does not have final bit; instead, returns last entry
*/
static inline struct dpaa2_sg_entry *
sg_to_qm_sg(struct scatterlist *sg, int sg_count,
sg_to_qm_sg(struct scatterlist *sg, int len,
struct dpaa2_sg_entry *qm_sg_ptr, u16 offset)
{
while (sg_count && sg) {
dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg),
sg_dma_len(sg), offset);
int ent_len;
while (len) {
ent_len = min_t(int, sg_dma_len(sg), len);
dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len,
offset);
qm_sg_ptr++;
sg = sg_next(sg);
sg_count--;
len -= ent_len;
}
return qm_sg_ptr - 1;
}
@ -42,11 +46,11 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count,
* convert scatterlist to h/w link table format
* scatterlist must have been previously dma mapped
*/
static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count,
static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len,
struct dpaa2_sg_entry *qm_sg_ptr,
u16 offset)
{
qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset);
qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset);
dpaa2_sg_set_final(qm_sg_ptr, true);
}

View File

@ -35,11 +35,9 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset &
SEC4_SG_OFFSET_MASK);
}
#ifdef DEBUG
print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ",
DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr,
sizeof(struct sec4_sg_entry), 1);
#endif
print_hex_dump_debug("sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4,
sec4_sg_ptr, sizeof(struct sec4_sg_entry), 1);
}
/*
@ -47,15 +45,19 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
* but does not have final bit; instead, returns last entry
*/
static inline struct sec4_sg_entry *
sg_to_sec4_sg(struct scatterlist *sg, int sg_count,
sg_to_sec4_sg(struct scatterlist *sg, int len,
struct sec4_sg_entry *sec4_sg_ptr, u16 offset)
{
while (sg_count) {
dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg),
sg_dma_len(sg), offset);
int ent_len;
while (len) {
ent_len = min_t(int, sg_dma_len(sg), len);
dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), ent_len,
offset);
sec4_sg_ptr++;
sg = sg_next(sg);
sg_count--;
len -= ent_len;
}
return sec4_sg_ptr - 1;
}
@ -72,11 +74,11 @@ static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr)
* convert scatterlist to h/w link table format
* scatterlist must have been previously dma mapped
*/
static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count,
static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int len,
struct sec4_sg_entry *sec4_sg_ptr,
u16 offset)
{
sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset);
sec4_sg_ptr = sg_to_sec4_sg(sg, len, sec4_sg_ptr, offset);
sg_to_sec4_set_last(sec4_sg_ptr);
}

View File

@ -7,7 +7,6 @@
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/authenc.h>
#include <crypto/crypto_wq.h>
#include <crypto/des.h>
#include <crypto/xts.h>
#include <linux/crypto.h>

View File

@ -1,4 +1,4 @@
// SPDX-License-Identifier: GPL-2.0
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_DEBUGFS_H
#define __NITROX_DEBUGFS_H

View File

@ -1,4 +1,4 @@
// SPDX-License-Identifier: GPL-2.0
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_MBX_H
#define __NITROX_MBX_H

View File

@ -2,7 +2,7 @@
/*
* AMD Cryptographic Coprocessor (CCP) AES crypto API support
*
* Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
* Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*/
@ -76,8 +76,7 @@ static int ccp_aes_crypt(struct ablkcipher_request *req, bool encrypt)
return -EINVAL;
if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) ||
(ctx->u.aes.mode == CCP_AES_MODE_CBC) ||
(ctx->u.aes.mode == CCP_AES_MODE_CFB)) &&
(ctx->u.aes.mode == CCP_AES_MODE_CBC)) &&
(req->nbytes & (AES_BLOCK_SIZE - 1)))
return -EINVAL;
@ -288,7 +287,7 @@ static struct ccp_aes_def aes_algs[] = {
.version = CCP_VERSION(3, 0),
.name = "cfb(aes)",
.driver_name = "cfb-aes-ccp",
.blocksize = AES_BLOCK_SIZE,
.blocksize = 1,
.ivsize = AES_BLOCK_SIZE,
.alg_defaults = &ccp_aes_defaults,
},

View File

@ -32,56 +32,62 @@ struct ccp_tasklet_data {
};
/* Human-readable error strings */
#define CCP_MAX_ERROR_CODE 64
static char *ccp_error_codes[] = {
"",
"ERR 01: ILLEGAL_ENGINE",
"ERR 02: ILLEGAL_KEY_ID",
"ERR 03: ILLEGAL_FUNCTION_TYPE",
"ERR 04: ILLEGAL_FUNCTION_MODE",
"ERR 05: ILLEGAL_FUNCTION_ENCRYPT",
"ERR 06: ILLEGAL_FUNCTION_SIZE",
"ERR 07: Zlib_MISSING_INIT_EOM",
"ERR 08: ILLEGAL_FUNCTION_RSVD",
"ERR 09: ILLEGAL_BUFFER_LENGTH",
"ERR 10: VLSB_FAULT",
"ERR 11: ILLEGAL_MEM_ADDR",
"ERR 12: ILLEGAL_MEM_SEL",
"ERR 13: ILLEGAL_CONTEXT_ID",
"ERR 14: ILLEGAL_KEY_ADDR",
"ERR 15: 0xF Reserved",
"ERR 16: Zlib_ILLEGAL_MULTI_QUEUE",
"ERR 17: Zlib_ILLEGAL_JOBID_CHANGE",
"ERR 18: CMD_TIMEOUT",
"ERR 19: IDMA0_AXI_SLVERR",
"ERR 20: IDMA0_AXI_DECERR",
"ERR 21: 0x15 Reserved",
"ERR 22: IDMA1_AXI_SLAVE_FAULT",
"ERR 23: IDMA1_AIXI_DECERR",
"ERR 24: 0x18 Reserved",
"ERR 25: ZLIBVHB_AXI_SLVERR",
"ERR 26: ZLIBVHB_AXI_DECERR",
"ERR 27: 0x1B Reserved",
"ERR 27: ZLIB_UNEXPECTED_EOM",
"ERR 27: ZLIB_EXTRA_DATA",
"ERR 30: ZLIB_BTYPE",
"ERR 31: ZLIB_UNDEFINED_SYMBOL",
"ERR 32: ZLIB_UNDEFINED_DISTANCE_S",
"ERR 33: ZLIB_CODE_LENGTH_SYMBOL",
"ERR 34: ZLIB _VHB_ILLEGAL_FETCH",
"ERR 35: ZLIB_UNCOMPRESSED_LEN",
"ERR 36: ZLIB_LIMIT_REACHED",
"ERR 37: ZLIB_CHECKSUM_MISMATCH0",
"ERR 38: ODMA0_AXI_SLVERR",
"ERR 39: ODMA0_AXI_DECERR",
"ERR 40: 0x28 Reserved",
"ERR 41: ODMA1_AXI_SLVERR",
"ERR 42: ODMA1_AXI_DECERR",
"ERR 43: LSB_PARITY_ERR",
"ILLEGAL_ENGINE",
"ILLEGAL_KEY_ID",
"ILLEGAL_FUNCTION_TYPE",
"ILLEGAL_FUNCTION_MODE",
"ILLEGAL_FUNCTION_ENCRYPT",
"ILLEGAL_FUNCTION_SIZE",
"Zlib_MISSING_INIT_EOM",
"ILLEGAL_FUNCTION_RSVD",
"ILLEGAL_BUFFER_LENGTH",
"VLSB_FAULT",
"ILLEGAL_MEM_ADDR",
"ILLEGAL_MEM_SEL",
"ILLEGAL_CONTEXT_ID",
"ILLEGAL_KEY_ADDR",
"0xF Reserved",
"Zlib_ILLEGAL_MULTI_QUEUE",
"Zlib_ILLEGAL_JOBID_CHANGE",
"CMD_TIMEOUT",
"IDMA0_AXI_SLVERR",
"IDMA0_AXI_DECERR",
"0x15 Reserved",
"IDMA1_AXI_SLAVE_FAULT",
"IDMA1_AIXI_DECERR",
"0x18 Reserved",
"ZLIBVHB_AXI_SLVERR",
"ZLIBVHB_AXI_DECERR",
"0x1B Reserved",
"ZLIB_UNEXPECTED_EOM",
"ZLIB_EXTRA_DATA",
"ZLIB_BTYPE",
"ZLIB_UNDEFINED_SYMBOL",
"ZLIB_UNDEFINED_DISTANCE_S",
"ZLIB_CODE_LENGTH_SYMBOL",
"ZLIB _VHB_ILLEGAL_FETCH",
"ZLIB_UNCOMPRESSED_LEN",
"ZLIB_LIMIT_REACHED",
"ZLIB_CHECKSUM_MISMATCH0",
"ODMA0_AXI_SLVERR",
"ODMA0_AXI_DECERR",
"0x28 Reserved",
"ODMA1_AXI_SLVERR",
"ODMA1_AXI_DECERR",
};
void ccp_log_error(struct ccp_device *d, int e)
void ccp_log_error(struct ccp_device *d, unsigned int e)
{
dev_err(d->dev, "CCP error: %s (0x%x)\n", ccp_error_codes[e], e);
if (WARN_ON(e >= CCP_MAX_ERROR_CODE))
return;
if (e < ARRAY_SIZE(ccp_error_codes))
dev_err(d->dev, "CCP error %d: %s\n", e, ccp_error_codes[e]);
else
dev_err(d->dev, "CCP error %d: Unknown Error\n", e);
}
/* List of CCPs, CCP count, read-write access lock, and access functions

Some files were not shown because too many files have changed in this diff Show More