extractor_ipc_gnu.c (19976B)
1 /* 2 This file is part of libextractor. 3 Copyright (C) 2012 Vidyut Samanta and Christian Grothoff 4 5 libextractor is free software; you can redistribute it and/or modify 6 it under the terms of the GNU General Public License as published 7 by the Free Software Foundation; either version 3, or (at your 8 option) any later version. 9 10 libextractor is distributed in the hope that it will be useful, but 11 WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 General Public License for more details. 14 15 You should have received a copy of the GNU General Public License 16 along with libextractor; see the file COPYING. If not, write to the 17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 18 Boston, MA 02110-1301, USA. 19 */ 20 /** 21 * @file main/extractor_ipc_gnu.c 22 * @brief IPC with plugin for GNU/POSIX systems 23 * @author Christian Grothoff 24 */ 25 #include "platform.h" 26 #include "extractor.h" 27 #include "extractor_datasource.h" 28 #include "extractor_logging.h" 29 #include "extractor_plugin_main.h" 30 #include "extractor_plugins.h" 31 #include "extractor_ipc.h" 32 #include <dirent.h> 33 #include <sys/types.h> 34 #include <sys/wait.h> 35 #include <sys/shm.h> 36 #include <signal.h> 37 #include <spawn.h> 38 #include "extractor_plugpath.h" 39 40 /** 41 * The environment, passed on to the plugin helper. 42 */ 43 extern char **environ; 44 45 /** 46 * A shared memory resource (often shared with several 47 * other processes). 48 */ 49 struct EXTRACTOR_SharedMemory 50 { 51 /** 52 * Pointer to the mapped region of the shm (covers the whole shm) 53 */ 54 void *shm_ptr; 55 56 /** 57 * Allocated size of the shm 58 */ 59 size_t shm_size; 60 61 /** 62 * POSIX id of the shm into which data is uncompressed 63 */ 64 int shm_id; 65 66 /** 67 * Name of the shm 68 */ 69 char shm_name[MAX_SHM_NAME + 1]; 70 71 /** 72 * Reference counter describing how many references share this SHM. 73 */ 74 unsigned int rc; 75 76 }; 77 78 79 /** 80 * Definition of an IPC communication channel with 81 * some plugin. 82 */ 83 struct EXTRACTOR_Channel 84 { 85 86 /** 87 * Buffer for reading data from the plugin. 88 */ 89 char *mdata; 90 91 /** 92 * Size of the @e mdata buffer. 93 */ 94 size_t mdata_size; 95 96 /** 97 * Memory segment shared with this process. 98 */ 99 struct EXTRACTOR_SharedMemory *shm; 100 101 /** 102 * The plugin this channel is to communicate with. 103 */ 104 struct EXTRACTOR_PluginList *plugin; 105 106 /** 107 * Pipe used to communicate information to the plugin child process. 108 * NULL if not initialized. 109 */ 110 int cpipe_in; 111 112 /** 113 * Number of valid bytes in the channel's buffer. 114 */ 115 size_t size; 116 117 /** 118 * Pipe used to read information about extracted meta data from 119 * the plugin child process. -1 if not initialized. 120 */ 121 int cpipe_out; 122 123 /** 124 * Process ID of the child process for this plugin. 0 for none. 125 */ 126 pid_t cpid; 127 128 /** 129 * When the plugin process was started, in milliseconds (see now_ms()). 130 */ 131 uint64_t start_time; 132 133 /** 134 * Set once the plugin process has sent us anything. Until then it may 135 * still be loading the plugin and its libraries. 136 */ 137 int started; 138 139 }; 140 141 142 /** 143 * How long a plugin process may take to load its plugin before it is 144 * treated as blocked, in milliseconds. Loading libraries for the first 145 * time can take far longer than the 500 ms that LE otherwise tolerates. 146 * GStreamer, for one, scans all its plugins when its registry does not 147 * exist yet, which took 55 s right after installing Homebrew's GStreamer 148 * on macOS. It only saves the registry once the scan completes, so it 149 * was killed before that and would fail the same way every time. 150 */ 151 #define STARTUP_GRACE_MS 120000 152 153 154 /** 155 * Get a monotonic time stamp. 156 * 157 * @return current time in milliseconds since some arbitrary epoch 158 */ 159 static uint64_t 160 now_ms (void) 161 { 162 struct timespec ts; 163 164 if (0 != clock_gettime (CLOCK_MONOTONIC, 165 &ts)) 166 return 0; 167 return ((uint64_t) ts.tv_sec) * 1000 + ts.tv_nsec / 1000000; 168 } 169 170 171 /** 172 * Create a shared memory area. 173 * 174 * @param size size of the shared area 175 * @return NULL on error 176 */ 177 struct EXTRACTOR_SharedMemory * 178 EXTRACTOR_IPC_shared_memory_create_ (size_t size) 179 { 180 struct EXTRACTOR_SharedMemory *shm; 181 unsigned int attempts = 0; 182 183 if (NULL == (shm = malloc (sizeof (struct EXTRACTOR_SharedMemory)))) 184 { 185 LOG_STRERROR ("malloc"); 186 return NULL; 187 } 188 /* POSIX only guarantees portability for a shared memory object name 189 that consists of an initial slash followed by one or more 190 characters, none of which is a slash. The name is not a file 191 system path: NetBSD keeps POSIX shared memory in a tmpfs of its 192 own on /var/shm and rejects any name with an embedded slash with 193 EINVAL, so prefixing a directory such as /tmp made every call to 194 shm_open() fail there. 195 O_EXCL: an object of the same name may be left over from a process 196 that died before it could shm_unlink() it. Never reuse it: Darwin 197 refuses to ftruncate() an object that already has a size, which 198 would make every extraction of this process fail. */ 199 do 200 { 201 snprintf (shm->shm_name, 202 MAX_SHM_NAME, 203 "/LE-%u-%u", 204 getpid (), 205 (unsigned int) random ()); 206 shm->shm_id = shm_open (shm->shm_name, 207 O_RDWR | O_CREAT | O_EXCL, 208 S_IRUSR | S_IWUSR); 209 } 210 while ( (-1 == shm->shm_id) && 211 (EEXIST == errno) && 212 (++attempts < 16) ); 213 if (-1 == shm->shm_id) 214 { 215 LOG_STRERROR_FILE ("shm_open", 216 shm->shm_name); 217 free (shm); 218 return NULL; 219 } 220 if ( (0 != ftruncate (shm->shm_id, size)) || 221 (NULL == (shm->shm_ptr = mmap (NULL, 222 size, 223 PROT_WRITE, 224 MAP_SHARED, 225 shm->shm_id, 226 0))) || 227 (((void*) -1) == shm->shm_ptr) ) 228 { 229 LOG_STRERROR ("ftruncate/mmap"); 230 (void) close (shm->shm_id); 231 (void) shm_unlink (shm->shm_name); 232 free (shm); 233 return NULL; 234 } 235 shm->shm_size = size; 236 shm->rc = 0; 237 return shm; 238 } 239 240 241 /** 242 * Change the reference counter for this shm instance. 243 * 244 * @param shm instance to update 245 * @param delta value to change RC by 246 * @return new RC 247 */ 248 unsigned int 249 EXTRACTOR_IPC_shared_memory_change_rc_ (struct EXTRACTOR_SharedMemory *shm, 250 int delta) 251 { 252 shm->rc += delta; 253 return shm->rc; 254 } 255 256 257 /** 258 * Destroy shared memory area. 259 * 260 * @param shm memory area to destroy 261 * @return NULL on error 262 */ 263 void 264 EXTRACTOR_IPC_shared_memory_destroy_ (struct EXTRACTOR_SharedMemory *shm) 265 { 266 munmap (shm->shm_ptr, 267 shm->shm_size); 268 (void) close (shm->shm_id); 269 (void) shm_unlink (shm->shm_name); 270 free (shm); 271 } 272 273 274 /** 275 * Initialize shared memory area from data source. 276 * 277 * @param shm memory area to initialize 278 * @param ds data source to use for initialization 279 * @param off offset to use in data source 280 * @param size number of bytes to copy 281 * @return -1 on error, otherwise number of bytes copied 282 */ 283 ssize_t 284 EXTRACTOR_IPC_shared_memory_set_ (struct EXTRACTOR_SharedMemory *shm, 285 struct EXTRACTOR_Datasource *ds, 286 uint64_t off, 287 size_t size) 288 { 289 if (-1 == 290 EXTRACTOR_datasource_seek_ (ds, 291 off, 292 SEEK_SET)) 293 { 294 LOG ("Failed to set IPC memory due to seek error\n"); 295 return -1; 296 } 297 if (size > shm->shm_size) 298 size = shm->shm_size; 299 return EXTRACTOR_datasource_read_ (ds, 300 shm->shm_ptr, 301 size); 302 } 303 304 305 /** 306 * Query datasource for current position 307 * 308 * @param ds data source to query 309 * @return current position in the datasource or UINT64_MAX on error 310 */ 311 uint64_t 312 EXTRACTOR_datasource_get_pos_ (struct EXTRACTOR_Datasource *ds) 313 { 314 int64_t pos = EXTRACTOR_datasource_seek_ (ds, 315 0, 316 SEEK_CUR); 317 318 if (-1 == pos) 319 return UINT64_MAX; 320 return pos; 321 } 322 323 324 /** 325 * Start the process that runs @a plugin: the helper program, which 326 * then reads requests from @a in and writes replies to @a out. 327 * 328 * The helper is exec()ed rather than running the plugin in a forked 329 * copy of ourselves: if the application is multi-threaded, a fork()ed 330 * child may only call async-signal-safe functions, and loading a 331 * plugin is far from that. It could deadlock on a lock that another 332 * thread held at the time of the fork(), and on Darwin the 333 * Objective-C runtime kills such a child outright. 334 * 335 * @param plugin the plugin to run 336 * @param in pipe end the helper reads from 337 * @param out pipe end the helper writes to 338 * @param[out] pid set to the process ID of the helper 339 * @return 0 on success, otherwise an error number 340 */ 341 static int 342 spawn_helper (const struct EXTRACTOR_PluginList *plugin, 343 int in, 344 int out, 345 pid_t *pid) 346 { 347 char *helper; 348 char in_s[16]; 349 char out_s[16]; 350 char *argv[7]; 351 int ret; 352 353 if (NULL == plugin->libname) 354 return ENOENT; 355 if (NULL == (helper = EXTRACTOR_find_helper_ ())) 356 { 357 LOG ("Could not find `%s'\n", 358 PLUGINHELPER); 359 return ENOENT; 360 } 361 snprintf (in_s, 362 sizeof (in_s), 363 "%d", 364 in); 365 snprintf (out_s, 366 sizeof (out_s), 367 "%d", 368 out); 369 argv[0] = helper; 370 argv[1] = in_s; 371 argv[2] = out_s; 372 argv[3] = plugin->libname; 373 argv[4] = plugin->short_libname; 374 argv[5] = plugin->plugin_options; /* may be NULL, ending the list */ 375 argv[6] = NULL; 376 ret = posix_spawn (pid, 377 helper, 378 NULL, 379 NULL, 380 argv, 381 environ); 382 free (helper); 383 return ret; 384 } 385 386 387 /** 388 * Create a channel to communicate with a process wrapping 389 * the plugin of the given name. Starts the process as well. 390 * 391 * @param plugin the plugin 392 * @param shm memory to share with the process 393 * @return NULL on error, otherwise IPC channel 394 */ 395 struct EXTRACTOR_Channel * 396 EXTRACTOR_IPC_channel_create_ (struct EXTRACTOR_PluginList *plugin, 397 struct EXTRACTOR_SharedMemory *shm) 398 { 399 struct EXTRACTOR_Channel *channel; 400 int p1[2]; 401 int p2[2]; 402 pid_t pid; 403 struct InitMessage *init; 404 size_t slen; 405 int err; 406 static int sigpipe_ignored; 407 408 if (! sigpipe_ignored) 409 { 410 /* The cpipe_in end may be written to after the plugin child 411 has died; without ignoring SIGPIPE this would terminate the 412 host application. As the channel uses pipes (not sockets), 413 MSG_NOSIGNAL is not applicable, so we ignore SIGPIPE here. */ 414 signal (SIGPIPE, SIG_IGN); 415 sigpipe_ignored = 1; 416 } 417 if (NULL == (channel = malloc (sizeof (struct EXTRACTOR_Channel)))) 418 { 419 LOG_STRERROR ("malloc"); 420 return NULL; 421 } 422 channel->mdata_size = 1024; 423 if (NULL == (channel->mdata = malloc (channel->mdata_size))) 424 { 425 LOG_STRERROR ("malloc"); 426 free (channel); 427 return NULL; 428 } 429 channel->shm = shm; 430 channel->plugin = plugin; 431 channel->size = 0; 432 if (0 != pipe (p1)) 433 { 434 LOG_STRERROR ("pipe"); 435 free (channel->mdata); 436 free (channel); 437 return NULL; 438 } 439 if (0 != pipe (p2)) 440 { 441 LOG_STRERROR ("pipe"); 442 (void) close (p1[0]); 443 (void) close (p1[1]); 444 free (channel->mdata); 445 free (channel); 446 return NULL; 447 } 448 /* our ends of the pipes must not be inherited by this helper, nor by 449 those of the other plugins */ 450 (void) fcntl (p1[1], F_SETFD, FD_CLOEXEC); 451 (void) fcntl (p2[0], F_SETFD, FD_CLOEXEC); 452 if (0 != (err = spawn_helper (plugin, 453 p1[0], 454 p2[1], 455 &pid))) 456 { 457 errno = err; 458 LOG_STRERROR ("posix_spawn"); 459 (void) close (p1[0]); 460 (void) close (p1[1]); 461 (void) close (p2[0]); 462 (void) close (p2[1]); 463 free (channel->mdata); 464 free (channel); 465 return NULL; 466 } 467 (void) close (p1[0]); 468 (void) close (p2[1]); 469 channel->cpipe_in = p1[1]; 470 channel->cpipe_out = p2[0]; 471 channel->cpid = pid; 472 channel->start_time = now_ms (); 473 channel->started = 0; 474 slen = strlen (shm->shm_name) + 1; 475 if (NULL == (init = malloc (sizeof (struct InitMessage) + slen))) 476 { 477 LOG_STRERROR ("malloc"); 478 EXTRACTOR_IPC_channel_destroy_ (channel); 479 return NULL; 480 } 481 init->opcode = MESSAGE_INIT_STATE; 482 init->reserved = 0; 483 init->reserved2 = 0; 484 init->shm_name_length = slen; 485 init->shm_map_size = shm->shm_size; 486 memcpy (&init[1], shm->shm_name, slen); 487 if (sizeof (struct InitMessage) + slen != 488 EXTRACTOR_IPC_channel_send_ (channel, 489 init, 490 sizeof (struct InitMessage) + slen) ) 491 { 492 LOG ("Failed to send INIT_STATE message to plugin\n"); 493 EXTRACTOR_IPC_channel_destroy_ (channel); 494 free (init); 495 return NULL; 496 } 497 free (init); 498 return channel; 499 } 500 501 502 /** 503 * Destroy communication channel with a plugin/process. Also 504 * destroys the process. 505 * 506 * @param channel channel to communicate with the plugin 507 */ 508 void 509 EXTRACTOR_IPC_channel_destroy_ (struct EXTRACTOR_Channel *channel) 510 { 511 int status; 512 513 /* Close the pipes first: should the kill() not be permitted (for 514 example by AppArmor's signal mediation), the plugin still exits 515 once it reads end-of-file, so waitpid() does not block forever. */ 516 if (0 != close (channel->cpipe_out)) 517 LOG_STRERROR ("close"); 518 if (0 != close (channel->cpipe_in)) 519 LOG_STRERROR ("close"); 520 if (0 != kill (channel->cpid, SIGKILL)) 521 LOG_STRERROR ("kill"); 522 if (-1 == waitpid (channel->cpid, &status, 0)) 523 LOG_STRERROR ("waitpid"); 524 if (NULL != channel->plugin) 525 channel->plugin->channel = NULL; 526 free (channel->mdata); 527 free (channel); 528 } 529 530 531 /** 532 * Send data via the given IPC channel (blocking). 533 * 534 * @param channel channel to communicate with the plugin 535 * @param buf data to send 536 * @param size number of bytes in buf to send 537 * @return -1 on error, number of bytes sent on success 538 * (never does partial writes) 539 */ 540 ssize_t 541 EXTRACTOR_IPC_channel_send_ (struct EXTRACTOR_Channel *channel, 542 const void *data, 543 size_t size) 544 { 545 const char *cdata = data; 546 size_t off = 0; 547 ssize_t ret; 548 549 while (off < size) 550 { 551 ret = write (channel->cpipe_in, &cdata[off], size - off); 552 if (ret <= 0) 553 { 554 if ((-1 == ret) && (EINTR == errno)) 555 continue; 556 if (-1 == ret) 557 LOG_STRERROR ("write"); 558 return -1; 559 } 560 off += ret; 561 } 562 return size; 563 } 564 565 566 /** 567 * Receive data from any of the given IPC channels (blocking). 568 * Wait for one of the plugins to reply. 569 * Selects on plugin output pipes, runs 'receive_reply' 570 * on each activated pipe until it gets a seek request 571 * or a done message. Called repeatedly by the user until all pipes are dry or 572 * broken. 573 * 574 * @param channels array of channels, channels that break may be set to NULL 575 * @param num_channels length of the @a channels array 576 * @param proc function to call to process messages (may be called 577 * more than once) 578 * @param proc_cls closure for @a proc 579 * @return -1 on error, 1 on success 580 */ 581 int 582 EXTRACTOR_IPC_channel_recv_ (struct EXTRACTOR_Channel **channels, 583 unsigned int num_channels, 584 EXTRACTOR_ChannelMessageProcessor proc, 585 void *proc_cls) 586 { 587 struct timeval tv; 588 fd_set to_check; 589 int max; 590 unsigned int i; 591 struct EXTRACTOR_Channel *channel; 592 ssize_t ret; 593 ssize_t iret; 594 char *ndata; 595 int closed_channel; 596 int starting_channel; 597 uint64_t now; 598 599 FD_ZERO (&to_check); 600 max = -1; 601 for (i = 0; i<num_channels; i++) 602 { 603 channel = channels[i]; 604 if (NULL == channel) 605 continue; 606 FD_SET (channel->cpipe_out, &to_check); 607 if (max < channel->cpipe_out) 608 max = channel->cpipe_out; 609 } 610 if (-1 == max) 611 { 612 return 1; /* nothing left to do! */ 613 } 614 tv.tv_sec = 0; 615 tv.tv_usec = 500000; /* 500 ms */ 616 if (0 >= select (max + 1, &to_check, NULL, NULL, &tv)) 617 { 618 /* an error or timeout -> something's wrong or all plugins hung up */ 619 closed_channel = 0; 620 starting_channel = 0; 621 now = now_ms (); 622 for (i = 0; i<num_channels; i++) 623 { 624 channel = channels[i]; 625 if (NULL == channel) 626 continue; 627 if (-1 == channel->plugin->seek_request) 628 { 629 if ( (! channel->started) && 630 (now - channel->start_time < STARTUP_GRACE_MS) ) 631 { 632 /* still loading the plugin, give it more time */ 633 starting_channel = 1; 634 continue; 635 } 636 /* plugin blocked for too long, kill channel */ 637 LOG ("Channel blocked, closing channel to %s\n", 638 channel->plugin->libname); 639 channel->plugin->channel = NULL; 640 channel->plugin->round_finished = 1; 641 EXTRACTOR_IPC_channel_destroy_ (channel); 642 channels[i] = NULL; 643 closed_channel = 1; 644 } 645 } 646 if ( (1 == closed_channel) || 647 (1 == starting_channel) ) 648 return 1; 649 /* strange, no channel is to blame, let's die just to be safe */ 650 if ((EINTR != errno) && (0 != errno)) 651 LOG_STRERROR ("select"); 652 return -1; 653 } 654 for (i = 0; i<num_channels; i++) 655 { 656 channel = channels[i]; 657 if (NULL == channel) 658 continue; 659 if (! FD_ISSET (channel->cpipe_out, &to_check)) 660 continue; 661 if (channel->mdata_size == channel->size) 662 { 663 /* not enough space, need to grow allocation (if allowed) */ 664 if (MAX_META_DATA == channel->mdata_size) 665 { 666 LOG ("Inbound message from channel too large, aborting\n"); 667 channel->plugin->channel = NULL; 668 channel->plugin->round_finished = 1; 669 EXTRACTOR_IPC_channel_destroy_ (channel); 670 channels[i] = NULL; 671 continue; 672 } 673 channel->mdata_size *= 2; 674 if (channel->mdata_size > MAX_META_DATA) 675 channel->mdata_size = MAX_META_DATA; 676 if (NULL == (ndata = realloc (channel->mdata, 677 channel->mdata_size))) 678 { 679 LOG_STRERROR ("realloc"); 680 channel->plugin->channel = NULL; 681 channel->plugin->round_finished = 1; 682 EXTRACTOR_IPC_channel_destroy_ (channel); 683 channels[i] = NULL; 684 continue; 685 } 686 channel->mdata = ndata; 687 } 688 do 689 { 690 iret = read (channel->cpipe_out, 691 &channel->mdata[channel->size], 692 channel->mdata_size - channel->size); 693 } 694 while ((-1 == iret) && (EINTR == errno)); 695 if ( (-1 == iret) || 696 (0 == iret) || 697 (-1 == (ret = EXTRACTOR_IPC_process_reply_ (channel->plugin, 698 channel->mdata, 699 channel->size + iret, 700 proc, proc_cls)) ) ) 701 { 702 if (-1 == iret) 703 LOG_STRERROR ("read"); 704 LOG ("Read error from channel, closing channel %s\n", 705 channel->plugin->libname); 706 /* The plugin is gone -- a crash in a plugin is exactly what this 707 library is built to survive. Forget the channel on the plugin 708 as well, or the next file would be handed to a 709 `struct EXTRACTOR_Channel' that has already been freed. */ 710 channel->plugin->channel = NULL; 711 channel->plugin->round_finished = 1; 712 EXTRACTOR_IPC_channel_destroy_ (channel); 713 channels[i] = NULL; 714 continue; 715 } 716 else 717 { 718 channel->started = 1; 719 channel->size = channel->size + iret - ret; 720 memmove (channel->mdata, 721 &channel->mdata[ret], 722 channel->size); 723 } 724 } 725 return 1; 726 } 727 728 729 /* end of extractor_ipc_gnu.c */