Cleanups.

Attempt to allocate the buffer before doing any decryption calls and, if
the attempt fails, report an error and return an error indication.

Make sizes unsigned, as they can't be negative;
EVP_CIPHER_CTX_block_size() returns an int, but that's just because they
picked the wrong data type in OpenSSL.

No need to use calloc() for the output buffer - just use malloc().

diff --git a/print-esp.c b/print-esp.c
index 38aa66632d76b2f789b449188655e90d76d15d96..5548bf186b2926174ed7f9bda26b5e599fff7426 100644 (file)
--- a/print-esp.c
+++ b/print-esp.c
@@ -156,9 +156,10 @@ int esp_print_decrypt_buffer_by_ikev2(netdissect_options *ndo,
 {
        struct sa_list *sa;
        const u_char *iv;
-       u_char *output_buffer;
-       int len, block_size, output_buffer_size;
+       unsigned int len;
        EVP_CIPHER_CTX *ctx;
+       unsigned int block_size, output_buffer_size;
+       u_char *output_buffer;
 
        /* initiator arg is any non-zero value */
        if(initiator) initiator=1;
@@ -189,17 +190,21 @@ int esp_print_decrypt_buffer_by_ikev2(netdissect_options *ndo,
        ctx = EVP_CIPHER_CTX_new();
        if (ctx == NULL)
                return 0;
+       /*
+        * Allocate a buffer for the decrypted data.
+        * The output buffer must be separate from the input buffer, and
+        * its size must be a multiple of the cipher block size.
+        */
+       block_size = (unsigned int)EVP_CIPHER_CTX_block_size(ctx);
+       output_buffer_size = len + (block_size - len % block_size);
+       output_buffer = (u_char *)malloc(output_buffer_size);
+       if (output_buffer == NULL) {
+               (*ndo->ndo_warning)(ndo, "can't allocate memory for decryption buffer");
+               return 0;
+       }
        if (EVP_CipherInit(ctx, sa->evp, sa->secret, NULL, 0) < 0)
                (*ndo->ndo_warning)(ndo, "espkey init failed");
        EVP_CipherInit(ctx, NULL, NULL, iv, 0);
-
-       /* We need a block size */
-       block_size = EVP_CIPHER_CTX_block_size(ctx);
-       /* We need the buffer size to be multiple of a block size */
-       output_buffer_size = len + (block_size - len % block_size);
-       output_buffer = (u_char *)calloc(output_buffer_size, sizeof(u_char));
-       /* EVP_Cipher output buffer should be different from the input one.
-        * Also it should be of size that is multiple of cipher block size. */
        EVP_Cipher(ctx, output_buffer, buf, len);
        EVP_CIPHER_CTX_free(ctx);
 
@@ -214,8 +219,6 @@ int esp_print_decrypt_buffer_by_ikev2(netdissect_options *ndo,
        ndo->ndo_snapend = end;
 
        return 1;
-
-
 }
 USES_APPLE_RST
 
@@ -623,8 +626,8 @@ esp_print(netdissect_options *ndo,
        const u_char *ivoff;
        const u_char *p;
        EVP_CIPHER_CTX *ctx;
+       unsigned int block_size, output_buffer_size;
        u_char *output_buffer;
-       int block_size, output_buffer_size;
 #endif
 
        esp = (const struct newesp *)bp;
@@ -722,7 +725,9 @@ esp_print(netdissect_options *ndo,
                ep = bp2 + len;
        }
 
+       /* pointer to the IV, if there is one */
        ivoff = (const u_char *)(esp + 1) + 0;
+       /* length of the IV, if there is one; 0, if there isn't */
        ivlen = sa->ivlen;
        secret = sa->secret;
        ep = ep - sa->authlen;
@@ -730,6 +735,20 @@ esp_print(netdissect_options *ndo,
        if (sa->evp) {
                ctx = EVP_CIPHER_CTX_new();
                if (ctx != NULL) {
+                       /*
+                        * Allocate a buffer for the decrypted data.
+                        * The output buffer must be separate from the
+                        * input buffer, and its size must be a multiple
+                        * of the cipher block size.
+                        */
+                       block_size = (unsigned int)EVP_CIPHER_CTX_block_size(ctx);
+                       output_buffer_size = len + (block_size - len % block_size);
+                       output_buffer = (u_char *)malloc(output_buffer_size);
+                       if (output_buffer == NULL) {
+                               (*ndo->ndo_warning)(ndo, "can't allocate memory for decryption buffer");
+                               return -1;
+                       }
+
                        if (EVP_CipherInit(ctx, sa->evp, secret, NULL, 0) < 0)
                                (*ndo->ndo_warning)(ndo, "espkey init failed");
 
@@ -737,13 +756,6 @@ esp_print(netdissect_options *ndo,
                        EVP_CipherInit(ctx, NULL, NULL, p, 0);
                        len = ep - (p + ivlen);
 
-                       /* We need a block size */
-                       block_size = EVP_CIPHER_CTX_block_size(ctx);
-                       /* We need the buffer size to be multiple of a block size */
-                       output_buffer_size = len + (block_size - len % block_size);
-                       output_buffer = (u_char *)calloc(output_buffer_size, sizeof(u_char));
-                       /* EVP_Cipher output buffer should be different from the input one.
-                        * Also it should be of size that is multiple of cipher block size. */
                        EVP_Cipher(ctx, output_buffer, p + ivlen, len);
                        EVP_CIPHER_CTX_free(ctx);
                        /*