C OpenSSL EVP_CipherInit_ex2 returns error for presumably options for aes-xts-plain64, when previously did not.

Ferocious1881 · 2026-10-27T19:57:35+0000

When trying to make a decryption subroutine for some luks header reader I got some problems, but later finalizing this minimal reproducible example here,

Code:

#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>

#include <openssl/core_names.h>
#include <openssl/params.h>
#include <openssl/evp.h>
#include <openssl/bio.h>
#include <openssl/err.h>

int main(int argc, char **argv) {
    int ret = 0, len;
    const uint64_t size = 258048;
    uint8_t *input_buffer = malloc(size);
    uint8_t *output_buffer = malloc(size);
    (void) memset(input_buffer, 0, size);
    (void) memset(output_buffer, 0, size);

    int32_t outlen = 0;
    uint8_t iv[1];
    uint32_t iv_length = 0;
    uint8_t key[64];
    uint32_t key_length = sizeof(key);
    static_assert(sizeof(key) == 64, "");
    (void) memset(key, 0, key_length);

    EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
    EVP_CIPHER *cipher = EVP_CIPHER_fetch(NULL, "AES-128-XTS", NULL);
    if (ctx == NULL || cipher == NULL) {
        goto err;
    }

    OSSL_PARAM params[3] = {
        OSSL_PARAM_construct_uint32(OSSL_CIPHER_PARAM_KEYLEN, &key_length),
        OSSL_PARAM_construct_uint32(OSSL_CIPHER_PARAM_IVLEN, &iv_length),
        OSSL_PARAM_construct_end()
    };

    if (!EVP_CipherInit_ex2(ctx, cipher, key, iv, 0, params)) {
        printf("LINE: %i\n", __LINE__);
        goto err;
    }
    if (!EVP_CipherUpdate(ctx, output_buffer, &outlen, input_buffer, size)) {
        printf("LINE: %i\n", __LINE__);
        goto err;
    }
    if (!EVP_CipherFinal_ex(ctx, output_buffer + outlen, &len)) {
        printf("LINE: %i\n", __LINE__);
        goto err;
    }
    ret = EXIT_SUCCESS;

    for (int x = 0; x < 16; x++) {
        for (int y = 0; y < 16; y++) {
            printf("%02x ", output_buffer[(x * 16) + y]);
        }
        printf("\n");
    }

err:
    EVP_CIPHER_free(cipher);
    EVP_CIPHER_CTX_free(ctx);
    return ret;
}

I compiled it as cc -g -O0 -lcrypto example.c -o example, and got the error code for an error in EVP_CipherInit_ex2 invocation when I ran the program.[/code]

covacat · 2026-10-27T20:12:46+0000

key length should be 32

Ferocious1881 · 2026-10-27T20:26:39+0000

covacat said:
key length should be 32

Rewoking it once again based on EVP_Cipher(3) as

Code:

#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>

#include <openssl/core_names.h>
#include <openssl/params.h>
#include <openssl/evp.h>
#include <openssl/bio.h>
#include <openssl/err.h>

int encrypt(const unsigned char *key, uint32_t key_length,
        const unsigned char *iv, uint32_t iv_length,
        const unsigned char *msg, size_t msg_len, unsigned char *out)
{
    int ret = 0, encrypt = 1, outlen, len;
    EVP_CIPHER_CTX *ctx = NULL;
    EVP_CIPHER *cipher = NULL;

    ctx = EVP_CIPHER_CTX_new();
    cipher = EVP_CIPHER_fetch(NULL, "AES-128-XTS", NULL);
    if (ctx == NULL || cipher == NULL)
        goto err;

    OSSL_PARAM params[3] = {
        OSSL_PARAM_construct_uint32(OSSL_CIPHER_PARAM_KEYLEN, &key_length),
        OSSL_PARAM_construct_uint32(OSSL_CIPHER_PARAM_IVLEN, &iv_length),
        OSSL_PARAM_construct_end()
    };
                                                                
    if (!EVP_CipherInit_ex2(ctx, cipher, key, iv, encrypt, params))
        goto err;

    if (!EVP_CipherUpdate(ctx, out, &outlen, msg, msg_len))
        goto err;
    if (!EVP_CipherFinal_ex(ctx, out + outlen, &len))
        goto err;
    ret = 1;
err:
    EVP_CIPHER_free(cipher);
    EVP_CIPHER_CTX_free(ctx);
    return ret;
}


int main(int argc, char **argv) {
    int ret = 0, len;
    const uint64_t size = 258048;
    uint8_t *input_buffer = malloc(size);
    uint8_t *output_buffer = malloc(size);
    (void) memset(input_buffer, 0, size);
    (void) memset(output_buffer, 0, size);

    int32_t outlen = 0;
    uint8_t iv[1];
    uint32_t iv_length = 0;
    uint8_t key[64];
    uint32_t key_length = sizeof(key);
    static_assert(sizeof(key) == 64, "");
    (void) memset(key, 0, key_length);

    encrypt(key, key_length, iv, key_length, input_buffer, size, output_buffer);

    for (int x = 0; x < 16; x++) {
        for (int y = 0; y < 16; y++) {
            printf("%02x ", output_buffer[(x * 16) + y]);
        }
        printf("\n");
    }

    return EXIT_SUCCESS;
}

does work, just got an output of pure zeros to console.

Ferocious1881 · 2026-10-27T20:34:40+0000

I one choose AES-256-XTS instead of AES-128-XTS does not remove the wall of zeros.

covacat · 2026-10-27T20:50:54+0000

because it errors out and encryption is never done
set key[0] = 'A' before encrypt()
first half of the key can not be the same as the 2nd half
from 512 bits key 256 is aes and 256 is xts or something
they cant be the same

covacat · 2026-10-27T20:52:24+0000

also at err: add this
if(!ret)
ERR_print_errors_fp(stderr);

Ferocious1881 · 2026-10-27T21:15:28+0000

covacat said:
also at err: add this
if(!ret)
ERR_print_errors_fp(stderr);

Invalid iv length. Do you know what iv uses the aes-xts-plain64 encryption by LUKS?, the specifications do not mention nothing about iv lengths.

covacat · 2026-10-27T21:24:28+0000

AI says 64 bits (8 bytes)

Ferocious1881 · 2026-10-27T21:35:45+0000

covacat said:
AI says 64 bits (8 bytes)

Content?

Ferocious1881 · 2026-10-27T21:38:46+0000

covacat said:
AI says 64 bits (8 bytes)

And now I have some weird error: digital envelope routines:evp_cipher_init_internal:invalid length

covacat · 2026-10-27T21:42:25+0000

key length is probably wrong

Ferocious1881 · 2026-10-27T21:50:57+0000

covacat said:
key length is probably wrong

Still same error with 64 bits key length!

Ferocious1881 · 2026-10-28T00:32:26+0000

After correcting the key length to 32 bits the iv length error did come back.

Ferocious1881 · 2026-10-28T01:05:02+0000

This is absurd this

Code:

#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>

#include <openssl/core_names.h>
#include <openssl/params.h>
#include <openssl/evp.h>
#include <openssl/bio.h>
#include <openssl/err.h>

int encrypt(const unsigned char *key, uint32_t key_length,
        const unsigned char *iv, uint32_t iv_length,
        const unsigned char *msg, size_t msg_len, unsigned char *out)
{
    int ret = 0, encrypt = 1, outlen, len;
    EVP_CIPHER_CTX *ctx = NULL;
    EVP_CIPHER *cipher = NULL;

    ctx = EVP_CIPHER_CTX_new();
    cipher = EVP_CIPHER_fetch(NULL, "AES-256-XTS", NULL);
    if (ctx == NULL || cipher == NULL)
        goto err;

    uint32_t iv_length_bits = 256;
    OSSL_PARAM params[3] = {
        OSSL_PARAM_construct_uint32(OSSL_CIPHER_PARAM_KEYLEN, &key_length),
        OSSL_PARAM_construct_end(),
        OSSL_PARAM_construct_uint(OSSL_CIPHER_PARAM_IVLEN, &iv_length_bits),
    };
                                                                
    if (!EVP_CipherInit_ex2(ctx, cipher, key, iv, encrypt, params))
        goto err;
    if (!EVP_CipherUpdate(ctx, out, &outlen, msg, msg_len))
        goto err;
    if (!EVP_CipherFinal_ex(ctx, out + outlen, &len))
        goto err;
    ret = 1;
err:
    if(!ret)
        ERR_print_errors_fp(stderr);
    EVP_CIPHER_free(cipher);
    EVP_CIPHER_CTX_free(ctx);
    return ret;
}


int main(int argc, char **argv) {
    int ret = 0, len;
    const uint64_t size = 258048;
    uint8_t *input_buffer = malloc(size);
    uint8_t *output_buffer = malloc(size);
    (void) memset(input_buffer, 0, size);
    (void) memset(output_buffer, 0, size);

    int32_t outlen = 0;
    uint8_t iv[16];
    uint32_t iv_length = sizeof(iv);
    static_assert(sizeof(iv) == 16, "");
    uint8_t key[64];
    (void) memset(key, 0, sizeof(key));
    uint32_t key_length = sizeof(key);
    for (int j = 0; j < 64; j++) {
        key[j] = j;
    }
    static_assert(sizeof(key) == 64, "");
    (void) memset(key, 0, key_length);

    encrypt(key, key_length, iv, key_length, input_buffer, size, output_buffer);

    for (int x = 0; x < 16; x++) {
        for (int y = 0; y < 16; y++) {
            printf("%02x ", output_buffer[(x * 16) + y]);
        }
        printf("\n");
    }

    return EXIT_SUCCESS;
}

is giving me the aes-xts duplicate keys error.
Well I am an idiot I did memset after my for loop.

Ferocious1881 · 2026-10-28T01:19:59+0000

Note for 128-bits the key length is 32 bytes, as /usr/src/crypto/openssl/providers/implementations/ciphers/cipher_aes_xts.c. But there is a mystery of why having OSSL_PARAM_construct_uint must have 16 as value or it does not work. And sorry I am an idiot.