ghiper.c (12676B)
1 /*************************************************************************** 2 * _ _ ____ _ 3 * Project ___| | | | _ \| | 4 * / __| | | | |_) | | 5 * | (__| |_| | _ <| |___ 6 * \___|\___/|_| \_\_____| 7 * 8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 9 * 10 * This software is licensed as described in the file COPYING, which 11 * you should have received as part of this distribution. The terms 12 * are also available at https://curl.se/docs/copyright.html. 13 * 14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell 15 * copies of the Software, and permit persons to whom the Software is 16 * furnished to do so, under the terms of the COPYING file. 17 * 18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 19 * KIND, either express or implied. 20 * 21 * SPDX-License-Identifier: curl 22 * 23 ***************************************************************************/ 24 /* <DESC> 25 * multi socket API usage together with glib2 26 * </DESC> 27 */ 28 /* Example application source code using the multi socket interface to 29 * download many files at once. 30 * 31 * Written by Jeff Pohlmeyer 32 33 Requires glib-2.x and a (POSIX?) system that has mkfifo(). 34 35 This is an adaptation of libcurl's "hipev.c" and libevent's "event-test.c" 36 sample programs, adapted to use glib's g_io_channel in place of libevent. 37 38 When running, the program creates the named pipe "hiper.fifo" 39 40 Whenever there is input into the fifo, the program reads the input as a list 41 of URL's and creates some new easy handles to fetch each URL via the 42 curl_multi "hiper" API. 43 44 45 Thus, you can try a single URL: 46 % echo http://www.yahoo.com > hiper.fifo 47 48 Or a whole bunch of them: 49 % cat my-url-list > hiper.fifo 50 51 The fifo buffer is handled almost instantly, so you can even add more URL's 52 while the previous requests are still being downloaded. 53 54 This is purely a demo app, all retrieved data is simply discarded by the write 55 callback. 56 57 */ 58 59 #include <glib.h> 60 #include <sys/stat.h> 61 #include <unistd.h> 62 #include <fcntl.h> 63 #include <stdlib.h> 64 #include <stdio.h> 65 #include <errno.h> 66 #include <curl/curl.h> 67 68 #define MSG_OUT g_print /* Change to "g_error" to write to stderr */ 69 #define SHOW_VERBOSE 0 /* Set to non-zero for libcurl messages */ 70 #define SHOW_PROGRESS 0 /* Set to non-zero to enable progress callback */ 71 72 /* Global information, common to all connections */ 73 struct GlobalInfo { 74 CURLM *multi; 75 guint timer_event; 76 int still_running; 77 }; 78 79 /* Information associated with a specific easy handle */ 80 struct ConnInfo { 81 CURL *easy; 82 char *url; 83 struct GlobalInfo *global; 84 char error[CURL_ERROR_SIZE]; 85 }; 86 87 /* Information associated with a specific socket */ 88 struct SockInfo { 89 curl_socket_t sockfd; 90 CURL *easy; 91 int action; 92 long timeout; 93 GIOChannel *ch; 94 guint ev; 95 struct GlobalInfo *global; 96 }; 97 98 /* Die if we get a bad CURLMcode somewhere */ 99 static void mcode_or_die(const char *where, CURLMcode code) 100 { 101 if(CURLM_OK != code) { 102 const char *s; 103 switch(code) { 104 case CURLM_BAD_HANDLE: s = "CURLM_BAD_HANDLE"; break; 105 case CURLM_BAD_EASY_HANDLE: s = "CURLM_BAD_EASY_HANDLE"; break; 106 case CURLM_OUT_OF_MEMORY: s = "CURLM_OUT_OF_MEMORY"; break; 107 case CURLM_INTERNAL_ERROR: s = "CURLM_INTERNAL_ERROR"; break; 108 case CURLM_BAD_SOCKET: s = "CURLM_BAD_SOCKET"; break; 109 case CURLM_UNKNOWN_OPTION: s = "CURLM_UNKNOWN_OPTION"; break; 110 case CURLM_LAST: s = "CURLM_LAST"; break; 111 default: s = "CURLM_unknown"; 112 } 113 MSG_OUT("ERROR: %s returns %s\n", where, s); 114 exit(code); 115 } 116 } 117 118 /* Check for completed transfers, and remove their easy handles */ 119 static void check_multi_info(struct GlobalInfo *g) 120 { 121 CURLMsg *msg; 122 int msgs_left; 123 124 MSG_OUT("REMAINING: %d\n", g->still_running); 125 while((msg = curl_multi_info_read(g->multi, &msgs_left))) { 126 if(msg->msg == CURLMSG_DONE) { 127 CURL *easy = msg->easy_handle; 128 CURLcode res = msg->data.result; 129 char *eff_url; 130 struct ConnInfo *conn; 131 curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn); 132 curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url); 133 MSG_OUT("DONE: %s => (%d) %s\n", eff_url, res, conn->error); 134 curl_multi_remove_handle(g->multi, easy); 135 free(conn->url); 136 curl_easy_cleanup(easy); 137 free(conn); 138 } 139 } 140 } 141 142 /* Called by glib when our timeout expires */ 143 static gboolean timer_cb(gpointer data) 144 { 145 struct GlobalInfo *g = (struct GlobalInfo *)data; 146 CURLMcode rc; 147 148 rc = curl_multi_socket_action(g->multi, 149 CURL_SOCKET_TIMEOUT, 0, &g->still_running); 150 mcode_or_die("timer_cb: curl_multi_socket_action", rc); 151 check_multi_info(g); 152 return FALSE; 153 } 154 155 /* Update the event timer after curl_multi library calls */ 156 static int update_timeout_cb(CURLM *multi, long timeout_ms, void *userp) 157 { 158 struct timeval timeout; 159 struct GlobalInfo *g = (struct GlobalInfo *)userp; 160 timeout.tv_sec = timeout_ms/1000; 161 timeout.tv_usec = (timeout_ms%1000)*1000; 162 163 MSG_OUT("*** update_timeout_cb %ld => %ld:%ld ***\n", 164 timeout_ms, timeout.tv_sec, timeout.tv_usec); 165 166 /* 167 * if timeout_ms is -1, just delete the timer 168 * 169 * For other values of timeout_ms, this should set or *update* the timer to 170 * the new value 171 */ 172 if(timeout_ms >= 0) 173 g->timer_event = g_timeout_add(timeout_ms, timer_cb, g); 174 return 0; 175 } 176 177 /* Called by glib when we get action on a multi socket */ 178 static gboolean event_cb(GIOChannel *ch, GIOCondition condition, gpointer data) 179 { 180 struct GlobalInfo *g = (struct GlobalInfo*) data; 181 CURLMcode rc; 182 int fd = g_io_channel_unix_get_fd(ch); 183 184 int action = 185 ((condition & G_IO_IN) ? CURL_CSELECT_IN : 0) | 186 ((condition & G_IO_OUT) ? CURL_CSELECT_OUT : 0); 187 188 rc = curl_multi_socket_action(g->multi, fd, action, &g->still_running); 189 mcode_or_die("event_cb: curl_multi_socket_action", rc); 190 191 check_multi_info(g); 192 if(g->still_running) { 193 return TRUE; 194 } 195 else { 196 MSG_OUT("last transfer done, kill timeout\n"); 197 if(g->timer_event) { 198 g_source_remove(g->timer_event); 199 } 200 return FALSE; 201 } 202 } 203 204 /* Clean up the SockInfo structure */ 205 static void remsock(struct SockInfo *f) 206 { 207 if(!f) { 208 return; 209 } 210 if(f->ev) { 211 g_source_remove(f->ev); 212 } 213 g_free(f); 214 } 215 216 /* Assign information to a SockInfo structure */ 217 static void setsock(struct SockInfo *f, curl_socket_t s, CURL *e, int act, 218 struct GlobalInfo *g) 219 { 220 GIOCondition kind = 221 ((act & CURL_POLL_IN) ? G_IO_IN : 0) | 222 ((act & CURL_POLL_OUT) ? G_IO_OUT : 0); 223 224 f->sockfd = s; 225 f->action = act; 226 f->easy = e; 227 if(f->ev) { 228 g_source_remove(f->ev); 229 } 230 f->ev = g_io_add_watch(f->ch, kind, event_cb, g); 231 } 232 233 /* Initialize a new SockInfo structure */ 234 static void addsock(curl_socket_t s, CURL *easy, int action, 235 struct GlobalInfo *g) 236 { 237 struct SockInfo *fdp = g_malloc0(sizeof(struct SockInfo)); 238 239 fdp->global = g; 240 fdp->ch = g_io_channel_unix_new(s); 241 setsock(fdp, s, easy, action, g); 242 curl_multi_assign(g->multi, s, fdp); 243 } 244 245 /* CURLMOPT_SOCKETFUNCTION */ 246 static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) 247 { 248 struct GlobalInfo *g = (struct GlobalInfo*) cbp; 249 struct SockInfo *fdp = (struct SockInfo*) sockp; 250 static const char *whatstr[]={ "none", "IN", "OUT", "INOUT", "REMOVE" }; 251 252 MSG_OUT("socket callback: s=%d e=%p what=%s ", s, e, whatstr[what]); 253 if(what == CURL_POLL_REMOVE) { 254 MSG_OUT("\n"); 255 remsock(fdp); 256 } 257 else { 258 if(!fdp) { 259 MSG_OUT("Adding data: %s%s\n", 260 (what & CURL_POLL_IN) ? "READ" : "", 261 (what & CURL_POLL_OUT) ? "WRITE" : ""); 262 addsock(s, e, what, g); 263 } 264 else { 265 MSG_OUT( 266 "Changing action from %d to %d\n", fdp->action, what); 267 setsock(fdp, s, e, what, g); 268 } 269 } 270 return 0; 271 } 272 273 /* CURLOPT_WRITEFUNCTION */ 274 static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *data) 275 { 276 size_t realsize = size * nmemb; 277 struct ConnInfo *conn = (struct ConnInfo*) data; 278 (void)ptr; 279 (void)conn; 280 return realsize; 281 } 282 283 /* CURLOPT_XFERINFOFUNCTION */ 284 static int xferinfo_cb(void *p, curl_off_t dltotal, curl_off_t dlnow, 285 curl_off_t ult, curl_off_t uln) 286 { 287 struct ConnInfo *conn = (struct ConnInfo *)p; 288 (void)ult; 289 (void)uln; 290 291 fprintf(MSG_OUT, "Progress: %s (%" CURL_FORMAT_CURL_OFF_T 292 "/%" CURL_FORMAT_CURL_OFF_T ")\n", conn->url, dlnow, dltotal); 293 return 0; 294 } 295 296 /* Create a new easy handle, and add it to the global curl_multi */ 297 static void new_conn(const char *url, struct GlobalInfo *g) 298 { 299 struct ConnInfo *conn; 300 CURLMcode rc; 301 302 conn = g_malloc0(sizeof(*conn)); 303 conn->error[0] = '\0'; 304 conn->easy = curl_easy_init(); 305 if(!conn->easy) { 306 MSG_OUT("curl_easy_init() failed, exiting!\n"); 307 exit(2); 308 } 309 conn->global = g; 310 conn->url = g_strdup(url); 311 curl_easy_setopt(conn->easy, CURLOPT_URL, conn->url); 312 curl_easy_setopt(conn->easy, CURLOPT_WRITEFUNCTION, write_cb); 313 curl_easy_setopt(conn->easy, CURLOPT_WRITEDATA, &conn); 314 curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, (long)SHOW_VERBOSE); 315 curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error); 316 curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn); 317 curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, SHOW_PROGRESS ? 0L : 1L); 318 curl_easy_setopt(conn->easy, CURLOPT_XFERINFOFUNCTION, xferinfo_cb); 319 curl_easy_setopt(conn->easy, CURLOPT_PROGRESSDATA, conn); 320 curl_easy_setopt(conn->easy, CURLOPT_FOLLOWLOCATION, 1L); 321 curl_easy_setopt(conn->easy, CURLOPT_CONNECTTIMEOUT, 30L); 322 curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_LIMIT, 1L); 323 curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_TIME, 30L); 324 325 MSG_OUT("Adding easy %p to multi %p (%s)\n", conn->easy, g->multi, url); 326 rc = curl_multi_add_handle(g->multi, conn->easy); 327 mcode_or_die("new_conn: curl_multi_add_handle", rc); 328 329 /* note that add_handle() sets a timeout to trigger soon so that the 330 necessary socket_action() gets called */ 331 } 332 333 /* This gets called by glib whenever data is received from the fifo */ 334 static gboolean fifo_cb(GIOChannel *ch, GIOCondition condition, gpointer data) 335 { 336 #define BUF_SIZE 1024 337 gsize len, tp; 338 gchar *buf, *tmp, *all = NULL; 339 GIOStatus rv; 340 341 do { 342 GError *err = NULL; 343 rv = g_io_channel_read_line(ch, &buf, &len, &tp, &err); 344 if(buf) { 345 if(tp) { 346 buf[tp]='\0'; 347 } 348 new_conn(buf, (struct GlobalInfo*)data); 349 g_free(buf); 350 } 351 else { 352 buf = g_malloc(BUF_SIZE + 1); 353 while(TRUE) { 354 buf[BUF_SIZE]='\0'; 355 g_io_channel_read_chars(ch, buf, BUF_SIZE, &len, &err); 356 if(len) { 357 buf[len]='\0'; 358 if(all) { 359 tmp = all; 360 all = g_strdup_printf("%s%s", tmp, buf); 361 g_free(tmp); 362 } 363 else { 364 all = g_strdup(buf); 365 } 366 } 367 else { 368 break; 369 } 370 } 371 if(all) { 372 new_conn(all, (struct GlobalInfo*)data); 373 g_free(all); 374 } 375 g_free(buf); 376 } 377 if(err) { 378 g_error("fifo_cb: %s", err->message); 379 g_free(err); 380 break; 381 } 382 } while((len) && (rv == G_IO_STATUS_NORMAL)); 383 return TRUE; 384 } 385 386 int init_fifo(void) 387 { 388 struct stat st; 389 const char *fifo = "hiper.fifo"; 390 int socket; 391 392 if(lstat(fifo, &st) == 0) { 393 if((st.st_mode & S_IFMT) == S_IFREG) { 394 errno = EEXIST; 395 perror("lstat"); 396 return CURL_SOCKET_BAD; 397 } 398 } 399 400 unlink(fifo); 401 if(mkfifo(fifo, 0600) == -1) { 402 perror("mkfifo"); 403 return CURL_SOCKET_BAD; 404 } 405 406 socket = open(fifo, O_RDWR | O_NONBLOCK, 0); 407 408 if(socket == CURL_SOCKET_BAD) { 409 perror("open"); 410 return socket; 411 } 412 MSG_OUT("Now, pipe some URL's into > %s\n", fifo); 413 414 return socket; 415 } 416 417 int main(void) 418 { 419 struct GlobalInfo *g = g_malloc0(sizeof(struct GlobalInfo)); 420 GMainLoop*gmain; 421 int fd; 422 GIOChannel* ch; 423 424 fd = init_fifo(); 425 if(fd == CURL_SOCKET_BAD) 426 return 1; 427 ch = g_io_channel_unix_new(fd); 428 g_io_add_watch(ch, G_IO_IN, fifo_cb, g); 429 gmain = g_main_loop_new(NULL, FALSE); 430 g->multi = curl_multi_init(); 431 curl_multi_setopt(g->multi, CURLMOPT_SOCKETFUNCTION, sock_cb); 432 curl_multi_setopt(g->multi, CURLMOPT_SOCKETDATA, g); 433 curl_multi_setopt(g->multi, CURLMOPT_TIMERFUNCTION, update_timeout_cb); 434 curl_multi_setopt(g->multi, CURLMOPT_TIMERDATA, g); 435 436 /* we do not call any curl_multi_socket*() function yet as we have no handles 437 added! */ 438 439 g_main_loop_run(gmain); 440 curl_multi_cleanup(g->multi); 441 return 0; 442 }