[gnutls-devel] gnutls_cipher_decrypt2() is broken for AES-GCM

[William McGovern]

Hello,

It seems that gnutls_cipher_decrypt2() does not work for the AES-GCM
ciphers and does not generate any data in the output buffer or return
any error code. In-place decryption of the same ciphertext with
gnutls_cipher_decrypt() works correctly.

This is seen with GnuTLS 3.1.6 on Mac OS X 10.8.2 as installed
by homebrew on platforms with and without AES-NI support.

I believe my code is correct but let me know if I missed anything :-)

- Will

Here is a test case that illustrates the defect:

/*
 * Test case illustrating bug in gnutls_cipher_decrypt2() with AES-GCM.
 *
 * VERSION: GnuTLS 3.1.6
 * 
 * DEFECT: The gnutls_cipher_decrypt2() function does not correctly decrypt
 *         data for the GNUTLS_CIPHER_AES_[128|256]_GCM
 *         ciphers. The output buffer for the decrypted data is not modified.
 *
 *  NOTES: In-place decryption using gnutls_cipher_decrypt() on the same
 *         ciphertext generated by gnutls_cipher_encrypt2() correctly
 *         decrypts the data in-place. Also, switching the cipher to CBC
 *         mode (and commenting out the AAD and AuthTag calls) shows that
 *         gnutls_cipher_decrypt2() works for CBC. It appears that the
 *         defect is isolated to the GCM mode with or without AES-NI.
 *
 *   ENVS: Mac OS X 10.8.2 : Intel Core i7-2600 (AES-NI)
 *         Mac OS X 10.8.2 : Intel Xeon X5482   (No AES-NI)
 *
 *  BUILD: cc -Wall -o test_case test_case.c -lgnutls
 *    RUN: ./test_case
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <gnutls/gnutls.h>
#include <gnutls/crypto.h>

#define CIPHER GNUTLS_CIPHER_AES_128_GCM

#define KEY_LEN   32
#define IV_LEN    12
#define AAD_LEN   64
#define DATA_LEN  1024
#define TAG_LEN   16

int
main(void)
{
    gnutls_cipher_hd_t h;
    gnutls_datum_t     keyDatum, ivDatum;
    void              *key, *iv, *aad, *pt1, *pt2, *ct, *tag1, *tag2;
    int                rc;

    key  = malloc(KEY_LEN);
    iv   = malloc(IV_LEN);
    aad  = malloc(AAD_LEN);
    pt1  = malloc(DATA_LEN);
    pt2  = malloc(DATA_LEN);
    ct   = malloc(DATA_LEN);
    tag1 = malloc(TAG_LEN);
    tag2 = malloc(TAG_LEN);
    assert(key  != NULL);
    assert(iv   != NULL);
    assert(aad  != NULL);
    assert(pt1  != NULL);
    assert(pt2  != NULL);
    assert(ct   != NULL);
    assert(tag1 != NULL);
    assert(tag2 != NULL);

    /* Construct datums for gnutls_cipher_init(). */
    keyDatum.data = key;
    keyDatum.size = KEY_LEN;
    ivDatum.data  = iv;
    ivDatum.size  = IV_LEN;

     /* Initialize input buffers (random) and zero output buffers. */
    gnutls_global_init();
    gnutls_rnd(GNUTLS_RND_RANDOM, key, KEY_LEN);
    gnutls_rnd(GNUTLS_RND_RANDOM, iv,  IV_LEN);
    gnutls_rnd(GNUTLS_RND_RANDOM, aad, AAD_LEN);
    gnutls_rnd(GNUTLS_RND_RANDOM, pt1, DATA_LEN);
    memset(pt2,  0, DATA_LEN);
    memset(tag1, 0, TAG_LEN);
    memset(tag2, 0, TAG_LEN);

    /* Encrypt data from pt1 -> ct */
    rc = gnutls_cipher_init(&h, CIPHER, &keyDatum, &ivDatum);
    assert(rc == 0);
    rc = gnutls_cipher_add_auth(h, aad, AAD_LEN);
    assert(rc == 0);
    rc = gnutls_cipher_encrypt2(h, pt1, DATA_LEN, ct, DATA_LEN);
    assert(rc == 0);
    rc = gnutls_cipher_tag(h, tag1, TAG_LEN);
    assert(rc == 0);
    gnutls_cipher_deinit(h);

    /* Decrypt data from ct -> pt2 */
    rc = gnutls_cipher_init(&h, CIPHER, &keyDatum, &ivDatum);
    assert(rc == 0);
    rc = gnutls_cipher_add_auth(h, aad, AAD_LEN);
    assert(rc == 0);
    rc = gnutls_cipher_decrypt2(h, ct, DATA_LEN, pt2, DATA_LEN);
    assert(rc == 0);
    rc = gnutls_cipher_tag(h, tag2, TAG_LEN);
    assert(rc == 0);
    gnutls_cipher_deinit(h);

    /* Verify decrypted data matches original plaintext (pt2 == pt1). */
    if (memcmp(pt1, pt2, DATA_LEN) != 0) {
        fprintf(stderr, "ERROR: Out-of-place decrypted data mismatch\n");
    } else {
        fprintf(stderr, "OK: Out-of-place decrypted data matches\n");
    }
    /* Verify authentication tags match. */
    if (memcmp(tag1, tag2, TAG_LEN) != 0) {
        fprintf(stderr, "ERROR: Out-of-place authentication tag mismatch\n");
    } else {
        fprintf(stderr, "OK: Out-of-place authentication tags match\n");
    }

    /* Decrypt data in-place in ct. */
    memset(tag2, 0, TAG_LEN);
    rc = gnutls_cipher_init(&h, CIPHER, &keyDatum, &ivDatum);
    assert(rc == 0);
    rc = gnutls_cipher_add_auth(h, aad, AAD_LEN);
    assert(rc == 0);
    rc = gnutls_cipher_decrypt(h, ct, DATA_LEN);
    assert(rc == 0);
    rc = gnutls_cipher_tag(h, tag2, TAG_LEN);
    assert(rc == 0);
    gnutls_cipher_deinit(h);

    /* Verify decrypted data matches original plaintext (pt2 == pt1). */
    if (memcmp(pt1, ct, DATA_LEN) != 0) {
        fprintf(stderr, "ERROR: In-place decrypted data mismatch\n");
    } else {
        fprintf(stderr, "OK: In-place decrypted data matches\n");
    }
 
    /* Verify authentication tags match. */
    if (memcmp(tag1, tag2, TAG_LEN) != 0) {
        fprintf(stderr, "ERROR: In-place authentication tag mismatch\n");
    } else {
        fprintf(stderr, "OK: In-place authentication tags match\n");
    }
    return 0;
}