twister

commit fb53910b8752495f3e2279cb7bd6ca68050145c9
parent 4c92c8aa27437a2f6b29749af187c0e15d13f83f
Author: Marcello Stanisci <stanisci.m@gmail.com>
Date:   Thu, 20 Dec 2018 19:35:58 +0100

Compression.

Adding compression logic, tested against curl CLI.

Diffstat:
M
src/test/test_twister.sh
 | 
39
+++++++++++++++++++++++++++++++++++++++
M
src/twister/taler-twister-service.c
 | 
96
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------------

2 files changed, 108 insertions(+), 27 deletions(-)
diff --git a/src/test/test_twister.sh b/src/test/test_twister.sh
@@ -1,4 +1,31 @@
 #!/bin/sh
+
+# This file is part of GNUnet.
+# Copyright (C) 2018 Taler Systems SA
+# 
+# Twister is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version
+# 3, or (at your option) any later version.
+# 
+# Twister is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
+# the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public
+# License along with Twister; see the file COPYING.  If not,
+# write to the Free Software Foundation, Inc., 51 Franklin
+# Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+# @author Marcello Stanisci
+# @author Christian Grothoff
+# 
+# @file test_twister.sh
+# 
+# @brief Twister testcases.
+
+
 if ! which curl > /dev/null
 then
     echo "curl not found";
@@ -117,6 +144,18 @@ if ! test '{"hello":"world"}' = $cum_mods; then
   exit 1
 fi
 
+# check if Twister does compression.
+status_code=$(curl -s "${TWISTER_URL}deflate-test" \
+  --compressed -o /dev/null -w "%{http_code}")
+
+# check status code is okay.
+if ! test 200 = $status_code; then
+  echo "Compression failed."
+  kill $web_server_pid
+  kill $twister_service_pid
+  exit 1
+fi
+
 # shutdown twister and webserver
 kill $web_server_pid
 kill $twister_service_pid
diff --git a/src/twister/taler-twister-service.c b/src/twister/taler-twister-service.c
@@ -422,12 +422,26 @@ curl_check_hdr (void *buffer,
                    hdr_type,
                    hdr_val);
 
-  if (0 == strcasecmp (hdr_type,
-                       MHD_HTTP_HEADER_CONTENT_ENCODING))
+  if ((0 == strcasecmp (hdr_type,
+                        MHD_HTTP_HEADER_CONTENT_ENCODING))
+      && (NULL != strstr (hdr_val,
+                          "deflate")))
   {
-    TALER_LOG_WARNING ("Artificially skipping 'C-E'\n");  
-    GNUNET_free (ndup);
-    return bytes;
+    TALER_LOG_INFO ("Compressed response\n");
+
+    hr->deflate = GNUNET_YES;
+    hr->zstate.zalloc = Z_NULL;
+    hr->zstate.zfree = Z_NULL;
+    hr->zstate.opaque = Z_NULL;
+
+    if (Z_OK != deflateInit (&hr->zstate,
+                             Z_DEFAULT_COMPRESSION))
+    {
+      TALER_LOG_ERROR ("Failed to initialize compression\n"); 
+      /* FIXME: put here a "emergency call"
+       * to suddenly return to the client.  */
+      return MHD_NO;
+    }
   }
 
   /* Skip "Host:" header as it will be wrong, given
@@ -883,31 +897,12 @@ con_val_iter (void *cls,
   (void) kind;
 
   if ((0 == strcmp (MHD_HTTP_HEADER_ACCEPT_ENCODING,
-                    key))
-      && (0 == strcmp ("deflate",
-                       value)))
+                    key)))
   {
-    hr->deflate = GNUNET_YES;
     TALER_LOG_INFO ("Compressed request\n");
     curl_easy_setopt (hr->curl,
                       CURLOPT_ENCODING,
-                      "deflate"); 
-    /**
-     * Init deflating structs here.
-     */
-    hr->zstate.zalloc = Z_NULL;
-    hr->zstate.zfree = Z_NULL;
-    hr->zstate.opaque = Z_NULL;
-
-    if (Z_OK != deflateInit (&hr->zstate,
-                             Z_DEFAULT_COMPRESSION))
-    {
-      TALER_LOG_ERROR ("Failed to initialize compression\n"); 
-      /* FIXME: put here a "emergency call" to suddenly return
-       * to the client.  */
-      return MHD_NO;
-    }
-
+                      value); 
     return MHD_YES;
   }
 
@@ -1706,8 +1701,55 @@ create_response (void *cls,
   }
 
   /**
-   * hr->io_len is to compress -- or compressed?
+   * COMPRESSION STARTS HERE.
    */
+  if (GNUNET_YES == hr->deflate)
+  {
+    #define CHUNK 1024
+    unsigned char out[CHUNK];
+    unsigned int uncompressed = hr->io_len;
+    unsigned int flush;
+    unsigned int done;
+    unsigned int zdone = 0;
+    unsigned int chunk;
+
+    TALER_LOG_DEBUG ("Compressing the response..\n"); 
+    do
+    {
+      chunk = GNUNET_MIN (CHUNK, uncompressed);
+      hr->zstate.avail_in = chunk;
+      hr->zstate.next_in = (unsigned char *)
+        &hr->io_buf[hr->io_len - uncompressed];
+      flush = CHUNK >= uncompressed ? Z_FINISH : Z_NO_FLUSH;
+
+      do
+      {
+        hr->zstate.avail_out = CHUNK;
+        hr->zstate.next_out = out;
+
+        GNUNET_assert
+          (Z_STREAM_ERROR != deflate (&hr->zstate, flush));
+        done = CHUNK - hr->zstate.avail_out;
+
+        /**
+         * No check for overflows, as compressing
+         * data yields always less bytes than the input.
+         */
+        TALER_LOG_DEBUG ("zdone/done: %u/%u\n",
+                         zdone,
+                         done);
+        memcpy (&hr->io_buf[zdone],
+                out,
+                done);
+        zdone += done;
+      }
+      while (0 != done);
+      TALER_LOG_DEBUG ("Out of inner compression loop\n");
+      uncompressed -= chunk;
+    }
+    while (Z_FINISH != flush);
+    deflateEnd (&hr->zstate);
+  }
 
   hr->response = MHD_create_response_from_buffer
     (hr->io_len,