base64: fix Curl_base64_encode and Curl_base64_decode interfaces

Previous interfaces for these libcurl internal functions did not allow to tell
apart a legitimate zero size result from an error condition. These functions
now return a CURLcode indicating function success or otherwise specific error.
Output size is returned using a pointer argument.

All usage of these two functions, and others closely related, has been adapted
to the new interfaces. Relative error and OOM handling adapted or added where
missing. Unit test 1302 also adapted.

1 files changed, 22 insertions, 11 deletions

diff --git a/lib/http_negotiate.c b/lib/http_negotiate.c
index b3d870c9c..f0cf16b9e 100644
--- a/lib/http_negotiate.c
+++ b/lib/http_negotiate.c
@@ -138,9 +138,11 @@ int Curl_input_negotiate(struct connectdata *conn, bool proxy,
   gss_buffer_desc input_token = GSS_C_EMPTY_BUFFER;
   gss_buffer_desc output_token = GSS_C_EMPTY_BUFFER;
   int ret;
-  size_t len, rawlen;
+  size_t len;
+  size_t rawlen = 0;
   bool gss;
   const char* protocol;
+  CURLcode error;
 
   while(*header && ISSPACE(*header))
     header++;
@@ -183,9 +185,9 @@ int Curl_input_negotiate(struct connectdata *conn, bool proxy,
 
   len = strlen(header);
   if(len > 0) {
-    rawlen = Curl_base64_decode(header,
-                                (unsigned char **)&input_token.value);
-    if(rawlen == 0)
+    error = Curl_base64_decode(header,
+                               (unsigned char **)&input_token.value, &rawlen);
+    if(error || rawlen == 0)
       return -1;
     input_token.length = rawlen;
 
@@ -270,8 +272,9 @@ CURLcode Curl_output_negotiate(struct connectdata *conn, bool proxy)
   struct negotiatedata *neg_ctx = proxy?&conn->data->state.proxyneg:
     &conn->data->state.negotiate;
   char *encoded = NULL;
-  size_t len;
+  size_t len = 0;
   char *userp;
+  CURLcode error;
 
 #ifdef HAVE_SPNEGO /* Handle SPNEGO */
   if(checkprefix("Negotiate", neg_ctx->protocol)) {
@@ -317,13 +320,21 @@ CURLcode Curl_output_negotiate(struct connectdata *conn, bool proxy)
     }
   }
 #endif
-  len = Curl_base64_encode(conn->data,
-                           neg_ctx->output_token.value,
-                           neg_ctx->output_token.length,
-                           &encoded);
+  error = Curl_base64_encode(conn->data,
+                             neg_ctx->output_token.value,
+                             neg_ctx->output_token.length,
+                             &encoded, &len);
+  if(error) {
+    Curl_safefree(neg_ctx->output_token.value);
+    neg_ctx->output_token.value = NULL;
+    return error;
+  }
 
-  if(len == 0)
-    return CURLE_OUT_OF_MEMORY;
+  if(len == 0) {
+    Curl_safefree(neg_ctx->output_token.value);
+    neg_ctx->output_token.value = NULL;
+    return CURLE_REMOTE_ACCESS_DENIED;
+  }
 
   userp = aprintf("%sAuthorization: %s %s\r\n", proxy ? "Proxy-" : "",
                   neg_ctx->protocol, encoded);