commit 0918af4e6851caf406075b5c8c0ec6f1387bc941
parent e5b20a5cce755a16019ed2b74151b9b8172d60e3
Author: Florian Dold <florian@dold.me>
Date: Mon, 9 Jan 2023 23:03:07 +0100
implement asynchronous http requests
Diffstat:
2 files changed, 310 insertions(+), 145 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,6 @@ tags
# compiled wallet core file
taler-wallet-core-qjs.mjs
cross/android-*.cross.txt
+
+build-*
+builddir
diff --git a/quickjs/quickjs-libc.c b/quickjs/quickjs-libc.c
@@ -154,9 +154,14 @@ typedef struct JSThreadState {
JSValue on_host_message_func;
JSHostMessageHandlerFn host_message_handler_f;
- void *host_message_handler_cls;
+ void *host_message_handler_cls;
int is_worker_thread;
+
+#ifndef NO_HTTP
+ CURLM *curlm;
+ struct list_head curl_requests;
+#endif
} JSThreadState;
static uint64_t os_pending_signals;
@@ -512,8 +517,6 @@ static JSValue js_std_writeFile(JSContext *ctx, JSValueConst this_val,
if (0 == sret) {
break;
}
- fprintf(stderr, "wrote %llu/%llu bytes\n", (unsigned long long) sret, (unsigned long long) data_len);
- fprintf(stderr, "requested %llu bytes\n", (unsigned long long) (data_len - bytes_written));
bytes_written += sret;
}
@@ -2100,12 +2103,20 @@ static void js_os_timer_mark(JSRuntime *rt, JSValueConst val,
#ifndef NO_HTTP
typedef struct {
- DynBuf response_data;
- JSValue headers_list;
- JSContext *ctx;
- uint8_t *readbuf;
- size_t readpos;
- size_t readlen;
+ // linked list of all requests
+ struct list_head link;
+ DynBuf response_data;
+ // curl request headers, must be kept around during the request
+ struct curl_slist *headers;
+ // Response headers
+ JSValue headers_list;
+ JSValue resolve_func;
+ JSValue reject_func;
+ JSContext* ctx;
+ uint8_t *readbuf;
+ size_t readpos;
+ size_t readlen;
+ CURL *curl;
} CurlRequestContext;
static size_t curl_header_callback(char *buffer, size_t size,
@@ -2199,152 +2210,217 @@ exception:
return -1;
}
-/**
- * fetchHttp(url, { method, headers, body })
- */
-static JSValue js_os_fetchHttp(JSContext *ctx, JSValueConst this_val,
- int argc, JSValueConst *argv)
+static void free_fetch_request_context(CurlRequestContext *req_context)
{
- const char *req_url = NULL;
- JSValue ret_val = JS_UNDEFINED;
- JSValue options = JS_UNINITIALIZED;
- JSValue method = JS_UNINITIALIZED;
- const char *method_str = NULL;
- CURLcode res;
- CURL *curl = NULL;
- CurlRequestContext req_context = { 0 };
- struct curl_slist *headers = NULL;
- int debug = FALSE;
-
- req_context.ctx = ctx;
- req_context.headers_list = JS_NewArray(ctx);
- dbuf_init(&req_context.response_data);
-
- req_url = JS_ToCString(ctx, argv[0]);
- if (!req_url) {
- goto exception;
- }
+ JSContext *ctx;
- options = argv[1];
- if (JS_VALUE_GET_TAG (options) == JS_TAG_UNDEFINED) {
- method = JS_NewString(ctx, "get");
- } else if (JS_VALUE_GET_TAG(options) == JS_TAG_OBJECT) {
- method = JS_GetPropertyStr(ctx, options, "method");
- debug = expect_property_str_bool (ctx, options, "debug");
- } else {
- JS_ThrowTypeError(ctx, "invalid options");
- goto exception;
- }
+ if (!req_context) {
+ return;
+ }
+ ctx = req_context->ctx;
+ req_context->ctx = NULL;
+ if (req_context->curl) {
+ curl_easy_cleanup(req_context->curl);
+ req_context->curl = NULL;
+ }
+ if (NULL != req_context->readbuf) {
+ free(req_context->readbuf);
+ }
+ dbuf_free(&req_context->response_data);
+ JS_FreeValue(ctx, req_context->headers_list);
+ JS_FreeValue(ctx, req_context->resolve_func);
+ JS_FreeValue(ctx, req_context->reject_func);
+ curl_slist_free_all(req_context->headers);
+ list_del(&req_context->link);
+ js_free(ctx, req_context);
+}
- curl = curl_easy_init();
- if (!curl) {
- JS_ThrowInternalError(ctx, "unable to init libcurl");
- goto exception;
- }
- curl_easy_setopt(curl, CURLOPT_URL, req_url);
- curl_easy_setopt(curl, CURLOPT_CAINFO, "/etc/ssl/certs/ca-certificates.crt");
- curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curl_header_callback);
- curl_easy_setopt(curl, CURLOPT_HEADERDATA, &req_context);
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb);
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, &req_context);
- if (debug == TRUE) {
- curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
- }
+static void
+finish_fetch_http(CurlRequestContext *req_context, CURLcode result)
+{
+ JSContext *ctx = req_context->ctx;
+ long resp_code;
+ JSValue ret_val;
+ JSValue cb_ret;
+
+ if (CURLE_OK != result) {
+ const char *errmsg;
+ JSAtom atom_message;
+
+ atom_message = JS_NewAtom(ctx, "message");
+ errmsg = curl_easy_strerror(result);
+ ret_val = JS_NewError(ctx);
+ JS_DefinePropertyValue(ctx, ret_val, atom_message,
+ JS_NewString(ctx, errmsg),
+ JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE);
+ cb_ret = JS_Call(ctx, req_context->reject_func, JS_UNDEFINED, 1, &ret_val);
+ JS_FreeAtom(ctx, atom_message);
+ JS_FreeValue(ctx, cb_ret);
+ JS_FreeValue(ctx, ret_val);
+ free_fetch_request_context(req_context);
+ return;
+ }
- // FIXME: This is only a temporary hack until we have proper TLS CA support
- // on all platforms
- curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
- curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
+ ret_val = JS_NewObject(ctx);
+ if (JS_IsException(ret_val)) {
+ // Huh, we're out of memory in the request handler?
+ fprintf(stderr, "fatal: can't allocate object in finish_fetch_http\n");
+ return;
+ }
- method_str = JS_ToCString(ctx, method);
+ curl_easy_getinfo(req_context->curl, CURLINFO_RESPONSE_CODE, &resp_code);
+ JS_SetPropertyStr(ctx, ret_val, "status", JS_NewInt32(ctx, resp_code));
+ JS_SetPropertyStr(ctx, ret_val, "headers", req_context->headers_list);
+ req_context->headers_list = JS_UNINITIALIZED;
+ // FIXME: Don't copy, own buffer
+ JS_SetPropertyStr(ctx,
+ ret_val,
+ "data",
+ JS_NewArrayBufferCopy(ctx,
+ req_context->response_data.buf,
+ req_context->response_data.size));
+ cb_ret = JS_Call(ctx, req_context->resolve_func, JS_UNDEFINED, 1, &ret_val);
+ JS_FreeValue(ctx, cb_ret);
+ JS_FreeValue(ctx, ret_val);
+ free_fetch_request_context(req_context);
+}
- if (0 == strcasecmp(method_str, "get")) {
- curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
- headers = curl_slist_append (headers, "Accept: application/json");
- } else if (0 == strcasecmp(method_str, "post")) {
- JSValue data;
- uint8_t *data_ptr;
- size_t data_len;
+/**
+ * fetchHttp(url, { method, headers, body }): Promise<Response>
+ */
+static JSValue js_os_fetchHttp(JSContext *ctx, JSValueConst this_val,
+ int argc, JSValueConst *argv)
+{
+ JSRuntime *rt = JS_GetRuntime(ctx);
+ JSThreadState *ts = JS_GetRuntimeOpaque(rt);
+ const char *req_url = NULL;
+ JSValue ret_val = JS_UNDEFINED;
+ JSValue resolving_funs[2];
+ JSValue options = JS_UNINITIALIZED;
+ JSValue method = JS_UNINITIALIZED;
+ const char *method_str = NULL;
+ CURLcode res;
+ CURLMcode mres;
+ CurlRequestContext *req_context = {0};
+ BOOL debug = FALSE;
+ BOOL req_started = FALSE;
+
+ req_context = js_mallocz(ctx, sizeof *req_context);
+
+ req_context->ctx = ctx;
+ req_context->headers_list = JS_NewArray(ctx);
+ dbuf_init(&req_context->response_data);
+
+ req_url = JS_ToCString(ctx, argv[0]);
+ if (!req_url) {
+ goto exception;
+ }
- data = JS_GetPropertyStr(ctx, options, "data");
- if (JS_IsException(data)) {
- goto exception;
+ options = argv[1];
+ if (JS_VALUE_GET_TAG(options) == JS_TAG_UNDEFINED) {
+ method = JS_NewString(ctx, "get");
}
- data_ptr = JS_GetArrayBuffer(ctx, &data_len, data);
- if (!data_ptr) {
- goto exception;
+ else if (JS_VALUE_GET_TAG(options) == JS_TAG_OBJECT) {
+ method = JS_GetPropertyStr(ctx, options, "method");
+ debug = expect_property_str_bool(ctx, options, "debug");
+ } else {
+ JS_ThrowTypeError(ctx, "invalid options");
+ goto exception;
}
- req_context.readbuf = malloc(data_len);
- memcpy(req_context.readbuf, data_ptr, data_len);
- req_context.readlen = data_len;
- JS_FreeValue(ctx, data);
- curl_easy_setopt(curl, CURLOPT_POST, 1L);
- curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback);
- curl_easy_setopt(curl, CURLOPT_READDATA, &req_context);
- headers = curl_slist_append (headers, "Content-Type: application/json");
- } else {
- JS_ThrowTypeError(ctx, "invalid request method");
- goto exception;
- }
- if (JS_VALUE_GET_TAG(options) == JS_TAG_OBJECT) {
- JSValue header_item = JS_GetPropertyStr(ctx, options, "headers");
- if (JS_IsException(header_item)) {
- goto exception;
+ req_context->curl = curl_easy_init();
+ if (!req_context->curl) {
+ JS_ThrowInternalError(ctx, "unable to init libcurl");
+ goto exception;
}
- if (JS_VALUE_GET_TAG(header_item) == JS_TAG_OBJECT) {
- if (0 != gather_headers(ctx, header_item , &headers)) {
- JS_FreeValue(ctx, header_item);
+ curl_easy_setopt(req_context->curl, CURLOPT_PRIVATE, req_context);
+ curl_easy_setopt(req_context->curl, CURLOPT_URL, req_url);
+ curl_easy_setopt(req_context->curl, CURLOPT_CAINFO, "/etc/ssl/certs/ca-certificates.crt");
+ curl_easy_setopt(req_context->curl, CURLOPT_HEADERFUNCTION, curl_header_callback);
+ curl_easy_setopt(req_context->curl, CURLOPT_HEADERDATA, req_context);
+ curl_easy_setopt(req_context->curl, CURLOPT_WRITEFUNCTION, curl_write_cb);
+ curl_easy_setopt(req_context->curl, CURLOPT_WRITEDATA, req_context);
+ if (debug == TRUE) {
+ curl_easy_setopt(req_context->curl, CURLOPT_VERBOSE, 1L);
+ }
+
+ // FIXME: This is only a temporary hack until we have proper TLS CA support
+ // on all platforms
+ curl_easy_setopt(req_context->curl, CURLOPT_SSL_VERIFYPEER, 0);
+ curl_easy_setopt(req_context->curl, CURLOPT_SSL_VERIFYHOST, 0);
+
+ method_str = JS_ToCString(ctx, method);
+
+ if (0 == strcasecmp(method_str, "get")) {
+ curl_easy_setopt(req_context->curl, CURLOPT_HTTPGET, 1L);
+ req_context->headers = curl_slist_append (req_context->headers, "Accept: application/json");
+ } else if (0 == strcasecmp(method_str, "post")) {
+ JSValue data;
+ uint8_t *data_ptr;
+ size_t data_len;
+
+ data = JS_GetPropertyStr(ctx, options, "data");
+ if (JS_IsException(data)) {
+ goto exception;
+ }
+ data_ptr = JS_GetArrayBuffer(ctx, &data_len, data);
+ if (!data_ptr) {
+ goto exception;
+ }
+ req_context->readbuf = malloc(data_len);
+ memcpy(req_context->readbuf, data_ptr, data_len);
+ req_context->readlen = data_len;
+ JS_FreeValue(ctx, data);
+ curl_easy_setopt(req_context->curl, CURLOPT_POST, 1L);
+ curl_easy_setopt(req_context->curl, CURLOPT_READFUNCTION, read_callback);
+ curl_easy_setopt(req_context->curl, CURLOPT_READDATA, req_context);
+ req_context->headers = curl_slist_append(req_context->headers, "Content-Type: application/json");
+ } else {
+ JS_ThrowTypeError(ctx, "invalid request method");
goto exception;
- }
}
- JS_FreeValue(ctx, header_item);
- }
- curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+ if (JS_VALUE_GET_TAG(options) == JS_TAG_OBJECT) {
+ JSValue header_item = JS_GetPropertyStr(ctx, options, "headers");
+ if (JS_IsException(header_item)) {
+ goto exception;
+ }
+ if (JS_VALUE_GET_TAG(header_item) == JS_TAG_OBJECT) {
+ if (0 != gather_headers(ctx, header_item, &req_context->headers)) {
+ JS_FreeValue(ctx, header_item);
+ goto exception;
+ }
+ }
+ JS_FreeValue(ctx, header_item);
+ }
+
+ curl_easy_setopt(req_context->curl, CURLOPT_HTTPHEADER, req_context->headers);
- res = curl_easy_perform(curl);
- if (CURLE_OK != res) {
- JS_ThrowInternalError(ctx, "fetch failed: %s", curl_easy_strerror (res));
- goto exception;
- }
- ret_val = JS_NewObject(ctx);
- if (JS_IsException(ret_val)) {
- goto exception;
- }
- {
- long resp_code;
+ mres = curl_multi_add_handle(ts->curlm, req_context->curl);
+ if (CURLM_OK != mres) {
+ JS_ThrowInternalError(ctx, "fetch failed: %s", curl_multi_strerror(mres));
+ goto exception;
+ }
+ list_add_tail(&req_context->link, &ts->curl_requests);
+ req_started = TRUE;
- curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp_code);
- JS_SetPropertyStr(ctx, ret_val, "status", JS_NewInt32(ctx, resp_code));
- JS_SetPropertyStr(ctx, ret_val, "headers", req_context.headers_list);
- req_context.headers_list = JS_UNINITIALIZED;
- // FIXME: Don't copy, own buffer
- JS_SetPropertyStr(ctx,
- ret_val,
- "data",
- JS_NewArrayBufferCopy(ctx,
- req_context.response_data.buf,
- req_context.response_data.size));
- }
+ ret_val = JS_NewPromiseCapability(ctx, resolving_funs);
+ if (JS_IsException(ret_val)) {
+ goto done;
+ }
+ req_context->resolve_func = resolving_funs[0];
+ req_context->reject_func = resolving_funs[1];
done:
- if (curl) {
- curl_easy_cleanup(curl);
- curl = NULL;
- }
- if (NULL != req_context.readbuf) {
- free(req_context.readbuf);
- }
- JS_FreeCString(ctx, method_str);
- dbuf_free(&req_context.response_data);
- JS_FreeValue(ctx, method);
- JS_FreeValue(ctx, req_context.headers_list);
- JS_FreeCString(ctx, req_url);
- curl_slist_free_all(headers);
- return ret_val;
+ if (FALSE == req_started) {
+ free_fetch_request_context(req_context);
+ }
+ JS_FreeValue(ctx, method);
+ JS_FreeCString(ctx, req_url);
+ JS_FreeCString(ctx, method_str);
+ return ret_val;
exception:
- ret_val = JS_EXCEPTION;
- goto done;
+ ret_val = JS_EXCEPTION;
+ goto done;
}
#endif
@@ -2622,16 +2698,51 @@ static int handle_host_message(JSRuntime *rt, JSContext *ctx)
return ret;
}
+// Perform curl network requests and
+static void do_curl(JSContext *ctx)
+{
+#ifndef NO_HTTP
+ JSRuntime *rt = JS_GetRuntime(ctx);
+ JSThreadState *ts = JS_GetRuntimeOpaque(rt);
+ CURLMcode curl_ret;
+ struct CURLMsg *m;
+ int running_handles;
+
+ curl_ret = curl_multi_perform(ts->curlm, &running_handles);
+ if (0 != curl_ret) {
+ fprintf(stderr, "curlm error: %s\n", curl_multi_strerror(curl_ret));
+ return;
+ }
+
+ do {
+ int msgq = 0;
+ m = curl_multi_info_read(ts->curlm, &msgq);
+ if (m && (m->msg == CURLMSG_DONE)) {
+ CurlRequestContext *req_ctx;
+ CURL *e = m->easy_handle;
+ CURLcode result = m->data.result;
+ if (CURLE_OK != curl_easy_getinfo(e, CURLINFO_PRIVATE, &req_ctx)) {
+ fprintf(stderr, "fatal: curl handle has no private data");
+ continue;
+ }
+ curl_multi_remove_handle(ts->curlm, e);
+ finish_fetch_http(req_ctx, result);
+ }
+ } while (m);
+#endif
+}
+
static int js_os_poll(JSContext *ctx)
{
JSRuntime *rt = JS_GetRuntime(ctx);
JSThreadState *ts = JS_GetRuntimeOpaque(rt);
int ret, fd_max, min_delay;
int64_t cur_time, delay;
- fd_set rfds, wfds;
+ fd_set rfds, wfds, ecxfds;
JSOSRWHandler *rh;
struct list_head *el;
- struct timeval tv, *tvp;
+ struct timeval tv = { 0 }, *tvp;
+ BOOL have_http_requests = FALSE;
/* only check signals in the main thread */
if (!ts->is_worker_thread &&
@@ -2650,9 +2761,14 @@ static int js_os_poll(JSContext *ctx)
}
}
- if (list_empty(&ts->os_rw_handlers) && list_empty(&ts->os_timers) &&
- list_empty(&ts->port_list) && JS_IsNull(ts->on_host_message_func))
+#ifndef NO_HTTP
+ have_http_requests = !list_empty(&ts->curl_requests);
+#endif
+
+ if ((!have_http_requests) && list_empty(&ts->os_rw_handlers) && list_empty(&ts->os_timers) &&
+ list_empty(&ts->port_list) && JS_IsNull(ts->on_host_message_func)) {
return -1; /* no more events */
+ }
if (!list_empty(&ts->os_timers)) {
cur_time = get_time_ms();
@@ -2684,6 +2800,7 @@ static int js_os_poll(JSContext *ctx)
FD_ZERO(&rfds);
FD_ZERO(&wfds);
+ FD_ZERO(&ecxfds);
fd_max = -1;
list_for_each(el, &ts->os_rw_handlers) {
rh = list_entry(el, JSOSRWHandler, link);
@@ -2706,6 +2823,37 @@ static int js_os_poll(JSContext *ctx)
fd_max = max_int(fd_max, ts->host_pipe->read_fd);
FD_SET(ts->host_pipe->read_fd, &rfds);
+#ifndef NO_HTTP
+ {
+ int curl_fd_max = -1;
+ CURLMcode mret;
+ long timeo = -1;
+
+ mret = curl_multi_fdset(ts->curlm, &rfds, &wfds, &ecxfds, &curl_fd_max);
+ if (CURLM_OK != mret) {
+ fprintf(stderr, "curlm error: %s\n", curl_multi_strerror(mret));
+ } else if (curl_fd_max != -1) {
+ fd_max = max_int(fd_max, curl_fd_max);
+ }
+
+ curl_multi_timeout(ts->curlm, &timeo);
+ if (timeo > 0) {
+ long timeo_sec = timeo / 1000;
+ long timeo_usec = (timeo % 1000) * 1000;
+ tvp = &tv;
+ if (tv.tv_sec < timeo_sec) {
+ tv.tv_sec = timeo_sec;
+ tv.tv_usec = timeo_usec;
+ } else if ((tv.tv_sec == timeo_sec) && tv.tv_usec < timeo_usec) {
+ tv.tv_usec = timeo_usec;
+ }
+ } else if (timeo == 0) {
+ do_curl(ctx);
+ goto done;
+ }
+ }
+#endif
+
ret = select(fd_max + 1, &rfds, &wfds, NULL, tvp);
if (ret > 0) {
list_for_each(el, &ts->os_rw_handlers) {
@@ -2740,6 +2888,8 @@ static int js_os_poll(JSContext *ctx)
goto done;
}
}
+
+ do_curl(ctx);
}
done:
return 0;
@@ -4357,6 +4507,11 @@ oom_fail:
JS_SetRuntimeOpaque(rt, ts);
+#ifndef NO_HTTP
+ ts->curlm = curl_multi_init();
+ init_list_head(&ts->curl_requests);
+#endif
+
#ifdef USE_WORKER
/* set the SharedArrayBuffer memory handlers */
{
@@ -4392,6 +4547,17 @@ void js_std_free_handlers(JSRuntime *rt)
free_timer(rt, th);
}
+#ifndef NO_HTTP
+
+ list_for_each_safe(el, el1, &ts->curl_requests) {
+ CurlRequestContext *request_ctx = list_entry(el, CurlRequestContext, link);
+ free_fetch_request_context(request_ctx);
+ }
+
+ curl_multi_cleanup(ts->curlm);
+ ts->curlm = NULL;
+#endif
+
JS_FreeValueRT(rt, ts->on_host_message_func);
#ifdef USE_WORKER
@@ -4416,7 +4582,6 @@ static void js_dump_obj(JSContext *ctx, FILE *f, JSValueConst val)
JS_FreeCString(ctx, str);
} else {
fprintf(f, "[exception]\n");
- abort();
}
}
@@ -4426,9 +4591,7 @@ static void js_std_dump_error1(JSContext *ctx, JSValueConst exception_val)
BOOL is_error;
is_error = JS_IsError(ctx, exception_val);
- fprintf(stderr, "dumping error, is_error: %u\n", is_error);
js_dump_obj(ctx, stderr, exception_val);
- fprintf(stderr, "dumped value\n");
if (is_error) {
val = JS_GetPropertyStr(ctx, exception_val, "stack");
if (!JS_IsUndefined(val)) {
@@ -4476,7 +4639,6 @@ void js_std_loop(JSContext *ctx)
}
if (!os_poll_func || os_poll_func(ctx)) {
- fprintf(stderr, "nothing more to wait on\n");
break;
}
}
