[gnutls-devel] gnutls_cipher_decrypt2() is broken for AES-GCM

[William McGovern]

Apologies if this is a double post. I sent this to bugs at gnutls.org but didn't
see the message show up on the list but maybe the list archiver has not
caught up yet...

Hello,

It seems that gnutls_cipher_decrypt2() does not work for the AES-GCM
ciphers and does not generate any data in the output buffer or return
any error code. In-place decryption of the same ciphertext with
gnutls_cipher_decrypt() works correctly.

This is seen with GnuTLS 3.1.6 on Mac OS X 10.8.2 as installed
by homebrew on platforms with and without AES-NI support.

I believe my code is correct but let me know if I missed anything :-)

- Will

Here is a test case that illustrates the defect:

/*
* Test case illustrating bug in gnutls_cipher_decrypt2() with AES-GCM.
*
* VERSION: GnuTLS 3.1.6
* 
* DEFECT: The gnutls_cipher_decrypt2() function does not correctly decrypt
*         data for the GNUTLS_CIPHER_AES_[128|256]_GCM
*         ciphers. The output buffer for the decrypted data is not modified.
*
*  NOTES: In-place decryption using gnutls_cipher_decrypt() on the same
*         ciphertext generated by gnutls_cipher_encrypt2() correctly
*         decrypts the data in-place. Also, switching the cipher to CBC
*         mode (and commenting out the AAD and AuthTag calls) shows that
*         gnutls_cipher_decrypt2() works for CBC. It appears that the
*         defect is isolated to the GCM mode with or without AES-NI.
*
*   ENVS: Mac OS X 10.8.2 : Intel Core i7-2600 (AES-NI)
*         Mac OS X 10.8.2 : Intel Xeon X5482   (No AES-NI)
*
*  BUILD: cc -Wall -o test_case test_case.c -lgnutls
*    RUN: ./test_case
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <gnutls/gnutls.h>
#include <gnutls/crypto.h>

#define CIPHER GNUTLS_CIPHER_AES_128_GCM

#define KEY_LEN   32
#define IV_LEN    12
#define AAD_LEN   64
#define DATA_LEN  1024
#define TAG_LEN   16

int
main(void)
{
   gnutls_cipher_hd_t h;
   gnutls_datum_t     keyDatum, ivDatum;
   void              *key, *iv, *aad, *pt1, *pt2, *ct, *tag1, *tag2;
   int                rc;

   key  = malloc(KEY_LEN);
   iv   = malloc(IV_LEN);
   aad  = malloc(AAD_LEN);
   pt1  = malloc(DATA_LEN);
   pt2  = malloc(DATA_LEN);
   ct   = malloc(DATA_LEN);
   tag1 = malloc(TAG_LEN);
   tag2 = malloc(TAG_LEN);
   assert(key  != NULL);
   assert(iv   != NULL);
   assert(aad  != NULL);
   assert(pt1  != NULL);
   assert(pt2  != NULL);
   assert(ct   != NULL);
   assert(tag1 != NULL);
   assert(tag2 != NULL);

   /* Construct datums for gnutls_cipher_init(). */
   keyDatum.data = key;
   keyDatum.size = KEY_LEN;
   ivDatum.data  = iv;
   ivDatum.size  = IV_LEN;

    /* Initialize input buffers (random) and zero output buffers. */
   gnutls_global_init();
   gnutls_rnd(GNUTLS_RND_RANDOM, key, KEY_LEN);
   gnutls_rnd(GNUTLS_RND_RANDOM, iv,  IV_LEN);
   gnutls_rnd(GNUTLS_RND_RANDOM, aad, AAD_LEN);
   gnutls_rnd(GNUTLS_RND_RANDOM, pt1, DATA_LEN);
   memset(pt2,  0, DATA_LEN);
   memset(tag1, 0, TAG_LEN);
   memset(tag2, 0, TAG_LEN);

   /* Encrypt data from pt1 -> ct */
   rc = gnutls_cipher_init(&h, CIPHER, &keyDatum, &ivDatum);
   assert(rc == 0);
   rc = gnutls_cipher_add_auth(h, aad, AAD_LEN);
   assert(rc == 0);
   rc = gnutls_cipher_encrypt2(h, pt1, DATA_LEN, ct, DATA_LEN);
   assert(rc == 0);
   rc = gnutls_cipher_tag(h, tag1, TAG_LEN);
   assert(rc == 0);
   gnutls_cipher_deinit(h);

   /* Decrypt data from ct -> pt2 */
   rc = gnutls_cipher_init(&h, CIPHER, &keyDatum, &ivDatum);
   assert(rc == 0);
   rc = gnutls_cipher_add_auth(h, aad, AAD_LEN);
   assert(rc == 0);
   rc = gnutls_cipher_decrypt2(h, ct, DATA_LEN, pt2, DATA_LEN);
   assert(rc == 0);
   rc = gnutls_cipher_tag(h, tag2, TAG_LEN);
   assert(rc == 0);
   gnutls_cipher_deinit(h);

   /* Verify decrypted data matches original plaintext (pt2 == pt1). */
   if (memcmp(pt1, pt2, DATA_LEN) != 0) {
       fprintf(stderr, "ERROR: Out-of-place decrypted data mismatch\n");
   } else {
       fprintf(stderr, "OK: Out-of-place decrypted data matches\n");
   }
   /* Verify authentication tags match. */
   if (memcmp(tag1, tag2, TAG_LEN) != 0) {
       fprintf(stderr, "ERROR: Out-of-place authentication tag mismatch\n");
   } else {
       fprintf(stderr, "OK: Out-of-place authentication tags match\n");
   }

   /* Decrypt data in-place in ct. */
   memset(tag2, 0, TAG_LEN);
   rc = gnutls_cipher_init(&h, CIPHER, &keyDatum, &ivDatum);
   assert(rc == 0);
   rc = gnutls_cipher_add_auth(h, aad, AAD_LEN);
   assert(rc == 0);
   rc = gnutls_cipher_decrypt(h, ct, DATA_LEN);
   assert(rc == 0);
   rc = gnutls_cipher_tag(h, tag2, TAG_LEN);
   assert(rc == 0);
   gnutls_cipher_deinit(h);

   /* Verify decrypted data matches original plaintext (pt2 == pt1). */
   if (memcmp(pt1, ct, DATA_LEN) != 0) {
       fprintf(stderr, "ERROR: In-place decrypted data mismatch\n");
   } else {
       fprintf(stderr, "OK: In-place decrypted data matches\n");
   }

   /* Verify authentication tags match. */
   if (memcmp(tag1, tag2, TAG_LEN) != 0) {
       fprintf(stderr, "ERROR: In-place authentication tag mismatch\n");
   } else {
       fprintf(stderr, "OK: In-place authentication tags match\n");
   }
   return 0;
}