proxy: Support HTTPS proxy and SOCKS+HTTP(s)

* HTTPS proxies:

An HTTPS proxy receives all transactions over an SSL/TLS connection.
Once a secure connection with the proxy is established, the user agent
uses the proxy as usual, including sending CONNECT requests to instruct
the proxy to establish a [usually secure] TCP tunnel with an origin
server. HTTPS proxies protect nearly all aspects of user-proxy
communications as opposed to HTTP proxies that receive all requests
(including CONNECT requests) in vulnerable clear text.

With HTTPS proxies, it is possible to have two concurrent _nested_
SSL/TLS sessions: the "outer" one between the user agent and the proxy
and the "inner" one between the user agent and the origin server
(through the proxy). This change adds supports for such nested sessions
as well.

A secure connection with a proxy requires its own set of the usual SSL
options (their actual descriptions differ and need polishing, see TODO):

  --proxy-cacert FILE        CA certificate to verify peer against
  --proxy-capath DIR         CA directory to verify peer against
  --proxy-cert CERT[:PASSWD] Client certificate file and password
  --proxy-cert-type TYPE     Certificate file type (DER/PEM/ENG)
  --proxy-ciphers LIST       SSL ciphers to use
  --proxy-crlfile FILE       Get a CRL list in PEM format from the file
  --proxy-insecure           Allow connections to proxies with bad certs
  --proxy-key KEY            Private key file name
  --proxy-key-type TYPE      Private key file type (DER/PEM/ENG)
  --proxy-pass PASS          Pass phrase for the private key
  --proxy-ssl-allow-beast    Allow security flaw to improve interop
  --proxy-sslv2              Use SSLv2
  --proxy-sslv3              Use SSLv3
  --proxy-tlsv1              Use TLSv1
  --proxy-tlsuser USER       TLS username
  --proxy-tlspassword STRING TLS password
  --proxy-tlsauthtype STRING TLS authentication type (default SRP)

All --proxy-foo options are independent from their --foo counterparts,
except --proxy-crlfile which defaults to --crlfile and --proxy-capath
which defaults to --capath.

Curl now also supports %{proxy_ssl_verify_result} --write-out variable,
similar to the existing %{ssl_verify_result} variable.

Supported backends: OpenSSL, GnuTLS, and NSS.

* A SOCKS proxy + HTTP/HTTPS proxy combination:

If both --socks* and --proxy options are given, Curl first connects to
the SOCKS proxy and then connects (through SOCKS) to the HTTP or HTTPS
proxy.

TODO: Update documentation for the new APIs and --proxy-* options.
Look for "Added in 7.XXX" marks.

1 files changed, 275 insertions, 42 deletions

diff --git a/lib/vtls/gskit.c b/lib/vtls/gskit.c
index 2ccb9e47b..a91d09cb7 100644
--- a/lib/vtls/gskit.c
+++ b/lib/vtls/gskit.c
@@ -81,6 +81,10 @@
 #include "memdebug.h"
 
 
+/* Directions. */
+#define SOS_READ        0x01
+#define SOS_WRITE       0x02
+
 /* SSL version flags. */
 #define CURL_GSKPROTO_SSLV2     0
 #define CURL_GSKPROTO_SSLV2_MASK        (1 << CURL_GSKPROTO_SSLV2)
@@ -289,10 +293,11 @@ static CURLcode set_callback(struct Curl_easy *data,
 }
 
 
-static CURLcode set_ciphers(struct Curl_easy *data,
+static CURLcode set_ciphers(struct connectdata *conn,
                                         gsk_handle h, unsigned int *protoflags)
 {
-  const char *cipherlist = data->set.str[STRING_SSL_CIPHER_LIST];
+  struct Curl_easy *data = conn->data;
+  const char *cipherlist = SSL_CONN_CONFIG(cipher_list);
   const char *clp;
   const gskit_cipher *ctp;
   int i;
@@ -340,7 +345,7 @@ static CURLcode set_ciphers(struct Curl_easy *data,
       break;
     /* Search the cipher in our table. */
     for(ctp = ciphertable; ctp->name; ctp++)
-      if(strnequal(ctp->name, clp, l) && !ctp->name[l])
+      if(strncasecompare(ctp->name, clp, l) && !ctp->name[l])
         break;
     if(!ctp->name) {
       failf(data, "Unknown cipher %.*s", l, clp);
@@ -500,17 +505,195 @@ static void close_async_handshake(struct ssl_connect_data *connssl)
   connssl->iocport = -1;
 }
 
+/* SSL over SSL
+ * Problems:
+ * 1) GSKit can only perform SSL on an AF_INET or AF_INET6 stream socket. To
+ *    pipe an SSL stream into another, it is therefore needed to have a pair
+ *    of such communicating sockets and handle the pipelining explicitly.
+ * 2) OS/400 socketpair() is only implemented for domain AF_UNIX, thus cannot
+ *    be used to produce the pipeline.
+ * The solution is to simulate socketpair() for AF_INET with low-level API
+ *    listen(), bind() and connect().
+ */
+
+static int
+inetsocketpair(int sv[2])
+{
+  int lfd;      /* Listening socket. */
+  int sfd;      /* Server socket. */
+  int cfd;      /* Client socket. */
+  int len;
+  struct sockaddr_in addr1;
+  struct sockaddr_in addr2;
+
+  /* Create listening socket on a local dynamic port. */
+  lfd = socket(AF_INET, SOCK_STREAM, 0);
+  if(lfd < 0)
+    return -1;
+  memset((char *) &addr1, 0, sizeof addr1);
+  addr1.sin_family = AF_INET;
+  addr1.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+  addr1.sin_port = 0;
+  if(bind(lfd, (struct sockaddr *) &addr1, sizeof addr1) ||
+     listen(lfd, 2) < 0) {
+    close(lfd);
+    return -1;
+  }
+
+  /* Get the allocated port. */
+  len = sizeof addr1;
+  if(getsockname(lfd, (struct sockaddr *) &addr1, &len) < 0) {
+    close(lfd);
+    return -1;
+  }
+
+  /* Create the client socket. */
+  cfd = socket(AF_INET, SOCK_STREAM, 0);
+  if(cfd < 0) {
+    close(lfd);
+    return -1;
+  }
+
+  /* Request unblocking connection to the listening socket. */
+  curlx_nonblock(cfd, TRUE);
+  if(connect(cfd, (struct sockaddr *) &addr1, sizeof addr1) < 0 &&
+     errno != EINPROGRESS) {
+    close(lfd);
+    close(cfd);
+    return -1;
+  }
+
+  /* Get the client dynamic port for intrusion check below. */
+  len = sizeof addr2;
+  if(getsockname(cfd, (struct sockaddr *) &addr2, &len) < 0) {
+    close(lfd);
+    close(cfd);
+    return -1;
+  }
+
+  /* Accept the incoming connection and get the server socket. */
+  curlx_nonblock(lfd, TRUE);
+  for(;;) {
+    len = sizeof addr1;
+    sfd = accept(lfd, (struct sockaddr *) &addr1, &len);
+    if(sfd < 0) {
+      close(lfd);
+      close(cfd);
+      return -1;
+    }
+
+    /* Check for possible intrusion from an external process. */
+    if(addr1.sin_addr.s_addr == addr2.sin_addr.s_addr &&
+       addr1.sin_port == addr2.sin_port)
+      break;
+
+    /* Intrusion: reject incoming connection. */
+    close(sfd);
+  }
+
+  /* Done, return sockets and succeed. */
+  close(lfd);
+  curlx_nonblock(cfd, FALSE);
+  sv[0] = cfd;
+  sv[1] = sfd;
+  return 0;
+}
+
+static int pipe_ssloverssl(struct connectdata *conn, int sockindex,
+                           int directions)
+{
+  struct ssl_connect_data *connssl = &conn->ssl[sockindex];
+  struct ssl_connect_data *connproxyssl = &conn->proxy_ssl[sockindex];
+  fd_set fds_read;
+  fd_set fds_write;
+  int n;
+  int m;
+  int i;
+  int ret = 0;
+  struct timeval tv = {0, 0};
+  char buf[CURL_MAX_WRITE_SIZE];
+
+  if(!connssl->use || !connproxyssl->use)
+    return 0;   /* No SSL over SSL: OK. */
+
+  FD_ZERO(&fds_read);
+  FD_ZERO(&fds_write);
+  n = -1;
+  if(directions & SOS_READ) {
+    FD_SET(connssl->remotefd, &fds_write);
+    n = connssl->remotefd;
+  }
+  if(directions & SOS_WRITE) {
+    FD_SET(connssl->remotefd, &fds_read);
+    n = connssl->remotefd;
+    FD_SET(conn->sock[sockindex], &fds_write);
+    if(n < conn->sock[sockindex])
+      n = conn->sock[sockindex];
+  }
+  i = select(n + 1, &fds_read, &fds_write, NULL, &tv);
+  if(i < 0)
+    return -1;  /* Select error. */
+
+  if(FD_ISSET(connssl->remotefd, &fds_write)) {
+    /* Try getting data from HTTPS proxy and pipe it upstream. */
+    n = 0;
+    i = gsk_secure_soc_read(connproxyssl->handle, buf, sizeof buf, &n);
+    switch(i) {
+    case GSK_OK:
+      if(n) {
+        i = write(connssl->remotefd, buf, n);
+        if(i < 0)
+          return -1;
+        ret = 1;
+      }
+      break;
+    case GSK_OS400_ERROR_TIMED_OUT:
+    case GSK_WOULD_BLOCK:
+      break;
+    default:
+      return -1;
+    }
+  }
+
+  if(FD_ISSET(connssl->remotefd, &fds_read) &&
+     FD_ISSET(conn->sock[sockindex], &fds_write)) {
+    /* Pipe data to HTTPS proxy. */
+    n = read(connssl->remotefd, buf, sizeof buf);
+    if(n < 0)
+      return -1;
+    if(n) {
+      i = gsk_secure_soc_write(connproxyssl->handle, buf, n, &m);
+      if(i != GSK_OK || n != m)
+        return -1;
+      ret = 1;
+    }
+  }
+
+  return ret;  /* OK */
+}
+
 
-static void close_one(struct ssl_connect_data *conn,
-                      struct Curl_easy *data)
+static void close_one(struct ssl_connect_data *connssl,
+                      struct connectdata *conn, int sockindex)
 {
-  if(conn->handle) {
-    gskit_status(data, gsk_secure_soc_close(&conn->handle),
+  if(connssl->handle) {
+    gskit_status(conn->data, gsk_secure_soc_close(&connssl->handle),
               "gsk_secure_soc_close()", 0);
-    conn->handle = (gsk_handle) NULL;
+    /* Last chance to drain output. */
+    while(pipe_ssloverssl(conn, sockindex, SOS_WRITE) > 0)
+      ;
+    connssl->handle = (gsk_handle) NULL;
+    if(connssl->localfd >= 0) {
+      close(connssl->localfd);
+      connssl->localfd = -1;
+    }
+    if(connssl->remotefd >= 0) {
+      close(connssl->remotefd);
+      connssl->remotefd = -1;
+    }
   }
-  if(conn->iocport >= 0)
-    close_async_handshake(conn);
+  if(connssl->iocport >= 0)
+    close_async_handshake(connssl);
 }
 
 
@@ -518,13 +701,18 @@ static ssize_t gskit_send(struct connectdata *conn, int sockindex,
                            const void *mem, size_t len, CURLcode *curlcode)
 {
   struct Curl_easy *data = conn->data;
-  CURLcode cc;
+  CURLcode cc = CURLE_SEND_ERROR;
   int written;
 
-  cc = gskit_status(data,
-                    gsk_secure_soc_write(conn->ssl[sockindex].handle,
-                                         (char *) mem, (int) len, &written),
-                    "gsk_secure_soc_write()", CURLE_SEND_ERROR);
+  if(pipe_ssloverssl(conn, sockindex, SOS_WRITE) >= 0) {
+    cc = gskit_status(data,
+                      gsk_secure_soc_write(conn->ssl[sockindex].handle,
+                                           (char *) mem, (int) len, &written),
+                      "gsk_secure_soc_write()", CURLE_SEND_ERROR);
+    if(cc == CURLE_OK)
+      if(pipe_ssloverssl(conn, sockindex, SOS_WRITE) < 0)
+        cc = CURLE_SEND_ERROR;
+  }
   if(cc != CURLE_OK) {
     *curlcode = cc;
     written = -1;
@@ -539,15 +727,23 @@ static ssize_t gskit_recv(struct connectdata *conn, int num, char *buf,
   struct Curl_easy *data = conn->data;
   int buffsize;
   int nread;
-  CURLcode cc;
+  CURLcode cc = CURLE_RECV_ERROR;
 
-  buffsize = buffersize > (size_t) INT_MAX? INT_MAX: (int) buffersize;
-  cc = gskit_status(data, gsk_secure_soc_read(conn->ssl[num].handle,
-                                              buf, buffsize, &nread),
-                    "gsk_secure_soc_read()", CURLE_RECV_ERROR);
-  if(cc != CURLE_OK) {
+  if(pipe_ssloverssl(conn, num, SOS_READ) >= 0) {
+    buffsize = buffersize > (size_t) INT_MAX? INT_MAX: (int) buffersize;
+    cc = gskit_status(data, gsk_secure_soc_read(conn->ssl[num].handle,
+                                                buf, buffsize, &nread),
+                      "gsk_secure_soc_read()", CURLE_RECV_ERROR);
+  }
+  switch(cc) {
+  case CURLE_OK:
+    break;
+  case CURLE_OPERATION_TIMEDOUT:
+    cc = CURLE_AGAIN;
+  default:
     *curlcode = cc;
     nread = -1;
+    break;
   }
   return (ssize_t) nread;
 }
@@ -560,18 +756,26 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex)
   gsk_handle envir;
   CURLcode result;
   int rc;
-  char *keyringfile;
-  char *keyringpwd;
-  char *keyringlabel;
-  char *sni;
+  const char * const keyringfile = SSL_CONN_CONFIG(CAfile);
+  const char * const keyringpwd = SSL_SET_OPTION(key_passwd);
+  const char * const keyringlabel = SSL_SET_OPTION(cert);
+  const long int ssl_version = SSL_CONN_CONFIG(version);
+  const bool verifypeer = SSL_CONN_CONFIG(verifypeer);
+  const char * const hostname = SSL_IS_PROXY()? conn->http_proxy.host.name:
+    conn->host.name;
+  const char *sni;
   unsigned int protoflags;
   long timeout;
   Qso_OverlappedIO_t commarea;
+  int sockpair[2];
+  static const int sobufsize = CURL_MAX_WRITE_SIZE;
 
   /* Create SSL environment, start (preferably asynchronous) handshake. */
 
   connssl->handle = (gsk_handle) NULL;
   connssl->iocport = -1;
+  connssl->localfd = -1;
+  connssl->remotefd = -1;
 
   /* GSKit supports two ways of specifying an SSL context: either by
    *  application identifier (that should have been defined at the system
@@ -586,9 +790,6 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex)
    *  application identifier mode is tried first, as recommended in IBM doc.
    */
 
-  keyringfile = data->set.str[STRING_SSL_CAFILE];
-  keyringpwd = data->set.str[STRING_KEY_PASSWD];
-  keyringlabel = data->set.str[STRING_CERT];
   envir = (gsk_handle) NULL;
 
   if(keyringlabel && *keyringlabel && !keyringpwd &&
@@ -613,16 +814,34 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex)
   if(result)
     return result;
 
+  /* Establish a pipelining socket pair for SSL over SSL. */
+  if(conn->proxy_ssl[sockindex].use) {
+    if(inetsocketpair(sockpair))
+      return CURLE_SSL_CONNECT_ERROR;
+    connssl->localfd = sockpair[0];
+    connssl->remotefd = sockpair[1];
+    setsockopt(connssl->localfd, SOL_SOCKET, SO_RCVBUF,
+               (void *) sobufsize, sizeof sobufsize);
+    setsockopt(connssl->remotefd, SOL_SOCKET, SO_RCVBUF,
+               (void *) sobufsize, sizeof sobufsize);
+    setsockopt(connssl->localfd, SOL_SOCKET, SO_SNDBUF,
+               (void *) sobufsize, sizeof sobufsize);
+    setsockopt(connssl->remotefd, SOL_SOCKET, SO_SNDBUF,
+               (void *) sobufsize, sizeof sobufsize);
+    curlx_nonblock(connssl->localfd, TRUE);
+    curlx_nonblock(connssl->remotefd, TRUE);
+  }
+
   /* Determine which SSL/TLS version should be enabled. */
   sni = conn->host.name;
-  switch (data->set.ssl.version) {
+  switch (ssl_version) {
   case CURL_SSLVERSION_SSLv2:
     protoflags = CURL_GSKPROTO_SSLV2_MASK;
-    sni = (char *) NULL;
+    sni = NULL;
     break;
   case CURL_SSLVERSION_SSLv3:
     protoflags = CURL_GSKPROTO_SSLV3_MASK;
-    sni = (char *) NULL;
+    sni = NULL;
     break;
   case CURL_SSLVERSION_DEFAULT:
   case CURL_SSLVERSION_TLSv1:
@@ -666,9 +885,12 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex)
                            (timeout + 999) / 1000);
   }
   if(!result)
-    result = set_numeric(data, connssl->handle, GSK_FD, conn->sock[sockindex]);
+    result = set_numeric(data, connssl->handle, GSK_OS400_READ_TIMEOUT, 1);
+  if(!result)
+    result = set_numeric(data, connssl->handle, GSK_FD, connssl->localfd >= 0?
+                         connssl->localfd: conn->sock[sockindex]);
   if(!result)
-    result = set_ciphers(data, connssl->handle, &protoflags);
+    result = set_ciphers(conn, connssl->handle, &protoflags);
   if(!protoflags) {
     failf(data, "No SSL protocol/cipher combination enabled");
     result = CURLE_SSL_CIPHER;
@@ -711,7 +933,7 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex)
   }
   if(!result)
     result = set_enum(data, connssl->handle, GSK_SERVER_AUTH_TYPE,
-                      data->set.ssl.verifypeer? GSK_SERVER_AUTH_FULL:
+                      verifypeer? GSK_SERVER_AUTH_FULL:
                       GSK_SERVER_AUTH_PASSTHRU, FALSE);
 
   if(!result) {
@@ -735,6 +957,10 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex)
     else if(errno != ENOBUFS)
       result = gskit_status(data, GSK_ERROR_IO,
                             "QsoCreateIOCompletionPort()", 0);
+    else if(conn->proxy_ssl[sockindex].use) {
+      /* Cannot pipeline while handshaking synchronously. */
+      result = CURLE_SSL_CONNECT_ERROR;
+    }
     else {
       /* No more completion port available. Use synchronous IO. */
       result = gskit_status(data, gsk_secure_soc_init(connssl->handle),
@@ -747,7 +973,7 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex)
   }
 
   /* Error: rollback. */
-  close_one(connssl, data);
+  close_one(connssl, conn, sockindex);
   return result;
 }
 
@@ -917,6 +1143,11 @@ static CURLcode gskit_connect_common(struct connectdata *conn, int sockindex,
       result = gskit_connect_step1(conn, sockindex);
   }
 
+  /* Handle handshake pipelining. */
+  if(!result)
+    if(pipe_ssloverssl(conn, sockindex, SOS_READ | SOS_WRITE) < 0)
+      result = CURLE_SSL_CONNECT_ERROR;
+
   /* Step 2: check if handshake is over. */
   if(!result && connssl->connecting_state == ssl_connect_2) {
     /* check allowed time left */
@@ -931,12 +1162,17 @@ static CURLcode gskit_connect_common(struct connectdata *conn, int sockindex,
       result = gskit_connect_step2(conn, sockindex, nonblocking);
   }
 
+  /* Handle handshake pipelining. */
+  if(!result)
+    if(pipe_ssloverssl(conn, sockindex, SOS_READ | SOS_WRITE) < 0)
+      result = CURLE_SSL_CONNECT_ERROR;
+
   /* Step 3: gather certificate info, verify host. */
   if(!result && connssl->connecting_state == ssl_connect_3)
     result = gskit_connect_step3(conn, sockindex);
 
   if(result)
-    close_one(connssl, data);
+    close_one(connssl, conn, sockindex);
   else if(connssl->connecting_state == ssl_connect_done) {
     connssl->state = ssl_connection_complete;
     connssl->connecting_state = ssl_connect_1;
@@ -980,11 +1216,8 @@ CURLcode Curl_gskit_connect(struct connectdata *conn, int sockindex)
 
 void Curl_gskit_close(struct connectdata *conn, int sockindex)
 {
-  struct Curl_easy *data = conn->data;
-  struct ssl_connect_data *connssl = &conn->ssl[sockindex];
-
-  if(connssl->use)
-    close_one(connssl, data);
+  close_one(&conn->ssl[sockindex], conn, sockindex);
+  close_one(&conn->proxy_ssl[sockindex], conn, sockindex);
 }
 
 
@@ -1003,7 +1236,7 @@ int Curl_gskit_shutdown(struct connectdata *conn, int sockindex)
   if(data->set.ftp_ccc != CURLFTPSSL_CCC_ACTIVE)
     return 0;
 
-  close_one(connssl, data);
+  close_one(connssl, conn, sockindex);
   rc = 0;
   what = SOCKET_READABLE(conn->sock[sockindex],
                          SSL_SHUTDOWN_TIMEOUT);