libextractor

GNU libextractor
Log | Files | Refs | Submodules | README | LICENSE

extractor_ipc_gnu.c (19976B)


      1 /*
      2      This file is part of libextractor.
      3      Copyright (C) 2012 Vidyut Samanta and Christian Grothoff
      4 
      5      libextractor is free software; you can redistribute it and/or modify
      6      it under the terms of the GNU General Public License as published
      7      by the Free Software Foundation; either version 3, or (at your
      8      option) any later version.
      9 
     10      libextractor is distributed in the hope that it will be useful, but
     11      WITHOUT ANY WARRANTY; without even the implied warranty of
     12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     13      General Public License for more details.
     14 
     15      You should have received a copy of the GNU General Public License
     16      along with libextractor; see the file COPYING.  If not, write to the
     17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
     18      Boston, MA 02110-1301, USA.
     19  */
     20 /**
     21  * @file main/extractor_ipc_gnu.c
     22  * @brief IPC with plugin for GNU/POSIX systems
     23  * @author Christian Grothoff
     24  */
     25 #include "platform.h"
     26 #include "extractor.h"
     27 #include "extractor_datasource.h"
     28 #include "extractor_logging.h"
     29 #include "extractor_plugin_main.h"
     30 #include "extractor_plugins.h"
     31 #include "extractor_ipc.h"
     32 #include <dirent.h>
     33 #include <sys/types.h>
     34 #include <sys/wait.h>
     35 #include <sys/shm.h>
     36 #include <signal.h>
     37 #include <spawn.h>
     38 #include "extractor_plugpath.h"
     39 
     40 /**
     41  * The environment, passed on to the plugin helper.
     42  */
     43 extern char **environ;
     44 
     45 /**
     46  * A shared memory resource (often shared with several
     47  * other processes).
     48  */
     49 struct EXTRACTOR_SharedMemory
     50 {
     51   /**
     52    * Pointer to the mapped region of the shm (covers the whole shm)
     53    */
     54   void *shm_ptr;
     55 
     56   /**
     57    * Allocated size of the shm
     58    */
     59   size_t shm_size;
     60 
     61   /**
     62    * POSIX id of the shm into which data is uncompressed
     63    */
     64   int shm_id;
     65 
     66   /**
     67    * Name of the shm
     68    */
     69   char shm_name[MAX_SHM_NAME + 1];
     70 
     71   /**
     72    * Reference counter describing how many references share this SHM.
     73    */
     74   unsigned int rc;
     75 
     76 };
     77 
     78 
     79 /**
     80  * Definition of an IPC communication channel with
     81  * some plugin.
     82  */
     83 struct EXTRACTOR_Channel
     84 {
     85 
     86   /**
     87    * Buffer for reading data from the plugin.
     88    */
     89   char *mdata;
     90 
     91   /**
     92    * Size of the @e mdata buffer.
     93    */
     94   size_t mdata_size;
     95 
     96   /**
     97    * Memory segment shared with this process.
     98    */
     99   struct EXTRACTOR_SharedMemory *shm;
    100 
    101   /**
    102    * The plugin this channel is to communicate with.
    103    */
    104   struct EXTRACTOR_PluginList *plugin;
    105 
    106   /**
    107    * Pipe used to communicate information to the plugin child process.
    108    * NULL if not initialized.
    109    */
    110   int cpipe_in;
    111 
    112   /**
    113    * Number of valid bytes in the channel's buffer.
    114    */
    115   size_t size;
    116 
    117   /**
    118    * Pipe used to read information about extracted meta data from
    119    * the plugin child process.  -1 if not initialized.
    120    */
    121   int cpipe_out;
    122 
    123   /**
    124    * Process ID of the child process for this plugin. 0 for none.
    125    */
    126   pid_t cpid;
    127 
    128   /**
    129    * When the plugin process was started, in milliseconds (see now_ms()).
    130    */
    131   uint64_t start_time;
    132 
    133   /**
    134    * Set once the plugin process has sent us anything.  Until then it may
    135    * still be loading the plugin and its libraries.
    136    */
    137   int started;
    138 
    139 };
    140 
    141 
    142 /**
    143  * How long a plugin process may take to load its plugin before it is
    144  * treated as blocked, in milliseconds.  Loading libraries for the first
    145  * time can take far longer than the 500 ms that LE otherwise tolerates.
    146  * GStreamer, for one, scans all its plugins when its registry does not
    147  * exist yet, which took 55 s right after installing Homebrew's GStreamer
    148  * on macOS. It only saves the registry once the scan completes, so it
    149  * was killed before that and would fail the same way every time.
    150  */
    151 #define STARTUP_GRACE_MS 120000
    152 
    153 
    154 /**
    155  * Get a monotonic time stamp.
    156  *
    157  * @return current time in milliseconds since some arbitrary epoch
    158  */
    159 static uint64_t
    160 now_ms (void)
    161 {
    162   struct timespec ts;
    163 
    164   if (0 != clock_gettime (CLOCK_MONOTONIC,
    165                           &ts))
    166     return 0;
    167   return ((uint64_t) ts.tv_sec) * 1000 + ts.tv_nsec / 1000000;
    168 }
    169 
    170 
    171 /**
    172  * Create a shared memory area.
    173  *
    174  * @param size size of the shared area
    175  * @return NULL on error
    176  */
    177 struct EXTRACTOR_SharedMemory *
    178 EXTRACTOR_IPC_shared_memory_create_ (size_t size)
    179 {
    180   struct EXTRACTOR_SharedMemory *shm;
    181   unsigned int attempts = 0;
    182 
    183   if (NULL == (shm = malloc (sizeof (struct EXTRACTOR_SharedMemory))))
    184   {
    185     LOG_STRERROR ("malloc");
    186     return NULL;
    187   }
    188   /* POSIX only guarantees portability for a shared memory object name
    189      that consists of an initial slash followed by one or more
    190      characters, none of which is a slash.  The name is not a file
    191      system path: NetBSD keeps POSIX shared memory in a tmpfs of its
    192      own on /var/shm and rejects any name with an embedded slash with
    193      EINVAL, so prefixing a directory such as /tmp made every call to
    194      shm_open() fail there.
    195      O_EXCL: an object of the same name may be left over from a process
    196      that died before it could shm_unlink() it.  Never reuse it: Darwin
    197      refuses to ftruncate() an object that already has a size, which
    198      would make every extraction of this process fail. */
    199   do
    200   {
    201     snprintf (shm->shm_name,
    202               MAX_SHM_NAME,
    203               "/LE-%u-%u",
    204               getpid (),
    205               (unsigned int) random ());
    206     shm->shm_id = shm_open (shm->shm_name,
    207                             O_RDWR | O_CREAT | O_EXCL,
    208                             S_IRUSR | S_IWUSR);
    209   }
    210   while ( (-1 == shm->shm_id) &&
    211           (EEXIST == errno) &&
    212           (++attempts < 16) );
    213   if (-1 == shm->shm_id)
    214   {
    215     LOG_STRERROR_FILE ("shm_open",
    216                        shm->shm_name);
    217     free (shm);
    218     return NULL;
    219   }
    220   if ( (0 != ftruncate (shm->shm_id, size)) ||
    221        (NULL == (shm->shm_ptr = mmap (NULL,
    222                                       size,
    223                                       PROT_WRITE,
    224                                       MAP_SHARED,
    225                                       shm->shm_id,
    226                                       0))) ||
    227        (((void*) -1) == shm->shm_ptr) )
    228   {
    229     LOG_STRERROR ("ftruncate/mmap");
    230     (void) close (shm->shm_id);
    231     (void) shm_unlink (shm->shm_name);
    232     free (shm);
    233     return NULL;
    234   }
    235   shm->shm_size = size;
    236   shm->rc = 0;
    237   return shm;
    238 }
    239 
    240 
    241 /**
    242  * Change the reference counter for this shm instance.
    243  *
    244  * @param shm instance to update
    245  * @param delta value to change RC by
    246  * @return new RC
    247  */
    248 unsigned int
    249 EXTRACTOR_IPC_shared_memory_change_rc_ (struct EXTRACTOR_SharedMemory *shm,
    250                                         int delta)
    251 {
    252   shm->rc += delta;
    253   return shm->rc;
    254 }
    255 
    256 
    257 /**
    258  * Destroy shared memory area.
    259  *
    260  * @param shm memory area to destroy
    261  * @return NULL on error
    262  */
    263 void
    264 EXTRACTOR_IPC_shared_memory_destroy_ (struct EXTRACTOR_SharedMemory *shm)
    265 {
    266   munmap (shm->shm_ptr,
    267           shm->shm_size);
    268   (void) close (shm->shm_id);
    269   (void) shm_unlink (shm->shm_name);
    270   free (shm);
    271 }
    272 
    273 
    274 /**
    275  * Initialize shared memory area from data source.
    276  *
    277  * @param shm memory area to initialize
    278  * @param ds data source to use for initialization
    279  * @param off offset to use in data source
    280  * @param size number of bytes to copy
    281  * @return -1 on error, otherwise number of bytes copied
    282  */
    283 ssize_t
    284 EXTRACTOR_IPC_shared_memory_set_ (struct EXTRACTOR_SharedMemory *shm,
    285                                   struct EXTRACTOR_Datasource *ds,
    286                                   uint64_t off,
    287                                   size_t size)
    288 {
    289   if (-1 ==
    290       EXTRACTOR_datasource_seek_ (ds,
    291                                   off,
    292                                   SEEK_SET))
    293   {
    294     LOG ("Failed to set IPC memory due to seek error\n");
    295     return -1;
    296   }
    297   if (size > shm->shm_size)
    298     size = shm->shm_size;
    299   return EXTRACTOR_datasource_read_ (ds,
    300                                      shm->shm_ptr,
    301                                      size);
    302 }
    303 
    304 
    305 /**
    306  * Query datasource for current position
    307  *
    308  * @param ds data source to query
    309  * @return current position in the datasource or UINT64_MAX on error
    310  */
    311 uint64_t
    312 EXTRACTOR_datasource_get_pos_ (struct EXTRACTOR_Datasource *ds)
    313 {
    314   int64_t pos = EXTRACTOR_datasource_seek_ (ds,
    315                                             0,
    316                                             SEEK_CUR);
    317 
    318   if (-1 == pos)
    319     return UINT64_MAX;
    320   return pos;
    321 }
    322 
    323 
    324 /**
    325  * Start the process that runs @a plugin: the helper program, which
    326  * then reads requests from @a in and writes replies to @a out.
    327  *
    328  * The helper is exec()ed rather than running the plugin in a forked
    329  * copy of ourselves: if the application is multi-threaded, a fork()ed
    330  * child may only call async-signal-safe functions, and loading a
    331  * plugin is far from that.  It could deadlock on a lock that another
    332  * thread held at the time of the fork(), and on Darwin the
    333  * Objective-C runtime kills such a child outright.
    334  *
    335  * @param plugin the plugin to run
    336  * @param in pipe end the helper reads from
    337  * @param out pipe end the helper writes to
    338  * @param[out] pid set to the process ID of the helper
    339  * @return 0 on success, otherwise an error number
    340  */
    341 static int
    342 spawn_helper (const struct EXTRACTOR_PluginList *plugin,
    343               int in,
    344               int out,
    345               pid_t *pid)
    346 {
    347   char *helper;
    348   char in_s[16];
    349   char out_s[16];
    350   char *argv[7];
    351   int ret;
    352 
    353   if (NULL == plugin->libname)
    354     return ENOENT;
    355   if (NULL == (helper = EXTRACTOR_find_helper_ ()))
    356   {
    357     LOG ("Could not find `%s'\n",
    358          PLUGINHELPER);
    359     return ENOENT;
    360   }
    361   snprintf (in_s,
    362             sizeof (in_s),
    363             "%d",
    364             in);
    365   snprintf (out_s,
    366             sizeof (out_s),
    367             "%d",
    368             out);
    369   argv[0] = helper;
    370   argv[1] = in_s;
    371   argv[2] = out_s;
    372   argv[3] = plugin->libname;
    373   argv[4] = plugin->short_libname;
    374   argv[5] = plugin->plugin_options; /* may be NULL, ending the list */
    375   argv[6] = NULL;
    376   ret = posix_spawn (pid,
    377                      helper,
    378                      NULL,
    379                      NULL,
    380                      argv,
    381                      environ);
    382   free (helper);
    383   return ret;
    384 }
    385 
    386 
    387 /**
    388  * Create a channel to communicate with a process wrapping
    389  * the plugin of the given name.  Starts the process as well.
    390  *
    391  * @param plugin the plugin
    392  * @param shm memory to share with the process
    393  * @return NULL on error, otherwise IPC channel
    394  */
    395 struct EXTRACTOR_Channel *
    396 EXTRACTOR_IPC_channel_create_ (struct EXTRACTOR_PluginList *plugin,
    397                                struct EXTRACTOR_SharedMemory *shm)
    398 {
    399   struct EXTRACTOR_Channel *channel;
    400   int p1[2];
    401   int p2[2];
    402   pid_t pid;
    403   struct InitMessage *init;
    404   size_t slen;
    405   int err;
    406   static int sigpipe_ignored;
    407 
    408   if (! sigpipe_ignored)
    409   {
    410     /* The cpipe_in end may be written to after the plugin child
    411        has died; without ignoring SIGPIPE this would terminate the
    412        host application.  As the channel uses pipes (not sockets),
    413        MSG_NOSIGNAL is not applicable, so we ignore SIGPIPE here. */
    414     signal (SIGPIPE, SIG_IGN);
    415     sigpipe_ignored = 1;
    416   }
    417   if (NULL == (channel = malloc (sizeof (struct EXTRACTOR_Channel))))
    418   {
    419     LOG_STRERROR ("malloc");
    420     return NULL;
    421   }
    422   channel->mdata_size = 1024;
    423   if (NULL == (channel->mdata = malloc (channel->mdata_size)))
    424   {
    425     LOG_STRERROR ("malloc");
    426     free (channel);
    427     return NULL;
    428   }
    429   channel->shm = shm;
    430   channel->plugin = plugin;
    431   channel->size = 0;
    432   if (0 != pipe (p1))
    433   {
    434     LOG_STRERROR ("pipe");
    435     free (channel->mdata);
    436     free (channel);
    437     return NULL;
    438   }
    439   if (0 != pipe (p2))
    440   {
    441     LOG_STRERROR ("pipe");
    442     (void) close (p1[0]);
    443     (void) close (p1[1]);
    444     free (channel->mdata);
    445     free (channel);
    446     return NULL;
    447   }
    448   /* our ends of the pipes must not be inherited by this helper, nor by
    449      those of the other plugins */
    450   (void) fcntl (p1[1], F_SETFD, FD_CLOEXEC);
    451   (void) fcntl (p2[0], F_SETFD, FD_CLOEXEC);
    452   if (0 != (err = spawn_helper (plugin,
    453                                 p1[0],
    454                                 p2[1],
    455                                 &pid)))
    456   {
    457     errno = err;
    458     LOG_STRERROR ("posix_spawn");
    459     (void) close (p1[0]);
    460     (void) close (p1[1]);
    461     (void) close (p2[0]);
    462     (void) close (p2[1]);
    463     free (channel->mdata);
    464     free (channel);
    465     return NULL;
    466   }
    467   (void) close (p1[0]);
    468   (void) close (p2[1]);
    469   channel->cpipe_in = p1[1];
    470   channel->cpipe_out = p2[0];
    471   channel->cpid = pid;
    472   channel->start_time = now_ms ();
    473   channel->started = 0;
    474   slen = strlen (shm->shm_name) + 1;
    475   if (NULL == (init = malloc (sizeof (struct InitMessage) + slen)))
    476   {
    477     LOG_STRERROR ("malloc");
    478     EXTRACTOR_IPC_channel_destroy_ (channel);
    479     return NULL;
    480   }
    481   init->opcode = MESSAGE_INIT_STATE;
    482   init->reserved = 0;
    483   init->reserved2 = 0;
    484   init->shm_name_length = slen;
    485   init->shm_map_size = shm->shm_size;
    486   memcpy (&init[1], shm->shm_name, slen);
    487   if (sizeof (struct InitMessage) + slen !=
    488       EXTRACTOR_IPC_channel_send_ (channel,
    489                                    init,
    490                                    sizeof (struct InitMessage) + slen) )
    491   {
    492     LOG ("Failed to send INIT_STATE message to plugin\n");
    493     EXTRACTOR_IPC_channel_destroy_ (channel);
    494     free (init);
    495     return NULL;
    496   }
    497   free (init);
    498   return channel;
    499 }
    500 
    501 
    502 /**
    503  * Destroy communication channel with a plugin/process.  Also
    504  * destroys the process.
    505  *
    506  * @param channel channel to communicate with the plugin
    507  */
    508 void
    509 EXTRACTOR_IPC_channel_destroy_ (struct EXTRACTOR_Channel *channel)
    510 {
    511   int status;
    512 
    513   /* Close the pipes first: should the kill() not be permitted (for
    514      example by AppArmor's signal mediation), the plugin still exits
    515      once it reads end-of-file, so waitpid() does not block forever. */
    516   if (0 != close (channel->cpipe_out))
    517     LOG_STRERROR ("close");
    518   if (0 != close (channel->cpipe_in))
    519     LOG_STRERROR ("close");
    520   if (0 != kill (channel->cpid, SIGKILL))
    521     LOG_STRERROR ("kill");
    522   if (-1 == waitpid (channel->cpid, &status, 0))
    523     LOG_STRERROR ("waitpid");
    524   if (NULL != channel->plugin)
    525     channel->plugin->channel = NULL;
    526   free (channel->mdata);
    527   free (channel);
    528 }
    529 
    530 
    531 /**
    532  * Send data via the given IPC channel (blocking).
    533  *
    534  * @param channel channel to communicate with the plugin
    535  * @param buf data to send
    536  * @param size number of bytes in buf to send
    537  * @return -1 on error, number of bytes sent on success
    538  *           (never does partial writes)
    539  */
    540 ssize_t
    541 EXTRACTOR_IPC_channel_send_ (struct EXTRACTOR_Channel *channel,
    542                              const void *data,
    543                              size_t size)
    544 {
    545   const char *cdata = data;
    546   size_t off = 0;
    547   ssize_t ret;
    548 
    549   while (off < size)
    550   {
    551     ret = write (channel->cpipe_in, &cdata[off], size - off);
    552     if (ret <= 0)
    553     {
    554       if ((-1 == ret) && (EINTR == errno))
    555         continue;
    556       if (-1 == ret)
    557         LOG_STRERROR ("write");
    558       return -1;
    559     }
    560     off += ret;
    561   }
    562   return size;
    563 }
    564 
    565 
    566 /**
    567  * Receive data from any of the given IPC channels (blocking).
    568  * Wait for one of the plugins to reply.
    569  * Selects on plugin output pipes, runs 'receive_reply'
    570  * on each activated pipe until it gets a seek request
    571  * or a done message. Called repeatedly by the user until all pipes are dry or
    572  * broken.
    573  *
    574  * @param channels array of channels, channels that break may be set to NULL
    575  * @param num_channels length of the @a channels array
    576  * @param proc function to call to process messages (may be called
    577  *             more than once)
    578  * @param proc_cls closure for @a proc
    579  * @return -1 on error, 1 on success
    580  */
    581 int
    582 EXTRACTOR_IPC_channel_recv_ (struct EXTRACTOR_Channel **channels,
    583                              unsigned int num_channels,
    584                              EXTRACTOR_ChannelMessageProcessor proc,
    585                              void *proc_cls)
    586 {
    587   struct timeval tv;
    588   fd_set to_check;
    589   int max;
    590   unsigned int i;
    591   struct EXTRACTOR_Channel *channel;
    592   ssize_t ret;
    593   ssize_t iret;
    594   char *ndata;
    595   int closed_channel;
    596   int starting_channel;
    597   uint64_t now;
    598 
    599   FD_ZERO (&to_check);
    600   max = -1;
    601   for (i = 0; i<num_channels; i++)
    602   {
    603     channel = channels[i];
    604     if (NULL == channel)
    605       continue;
    606     FD_SET (channel->cpipe_out, &to_check);
    607     if (max < channel->cpipe_out)
    608       max = channel->cpipe_out;
    609   }
    610   if (-1 == max)
    611   {
    612     return 1;   /* nothing left to do! */
    613   }
    614   tv.tv_sec = 0;
    615   tv.tv_usec = 500000; /* 500 ms */
    616   if (0 >= select (max + 1, &to_check, NULL, NULL, &tv))
    617   {
    618     /* an error or timeout -> something's wrong or all plugins hung up */
    619     closed_channel = 0;
    620     starting_channel = 0;
    621     now = now_ms ();
    622     for (i = 0; i<num_channels; i++)
    623     {
    624       channel = channels[i];
    625       if (NULL == channel)
    626         continue;
    627       if (-1 == channel->plugin->seek_request)
    628       {
    629         if ( (! channel->started) &&
    630              (now - channel->start_time < STARTUP_GRACE_MS) )
    631         {
    632           /* still loading the plugin, give it more time */
    633           starting_channel = 1;
    634           continue;
    635         }
    636         /* plugin blocked for too long, kill channel */
    637         LOG ("Channel blocked, closing channel to %s\n",
    638              channel->plugin->libname);
    639         channel->plugin->channel = NULL;
    640         channel->plugin->round_finished = 1;
    641         EXTRACTOR_IPC_channel_destroy_ (channel);
    642         channels[i] = NULL;
    643         closed_channel = 1;
    644       }
    645     }
    646     if ( (1 == closed_channel) ||
    647          (1 == starting_channel) )
    648       return 1;
    649     /* strange, no channel is to blame, let's die just to be safe */
    650     if ((EINTR != errno) && (0 != errno))
    651       LOG_STRERROR ("select");
    652     return -1;
    653   }
    654   for (i = 0; i<num_channels; i++)
    655   {
    656     channel = channels[i];
    657     if (NULL == channel)
    658       continue;
    659     if (! FD_ISSET (channel->cpipe_out, &to_check))
    660       continue;
    661     if (channel->mdata_size == channel->size)
    662     {
    663       /* not enough space, need to grow allocation (if allowed) */
    664       if (MAX_META_DATA == channel->mdata_size)
    665       {
    666         LOG ("Inbound message from channel too large, aborting\n");
    667         channel->plugin->channel = NULL;
    668         channel->plugin->round_finished = 1;
    669         EXTRACTOR_IPC_channel_destroy_ (channel);
    670         channels[i] = NULL;
    671         continue;
    672       }
    673       channel->mdata_size *= 2;
    674       if (channel->mdata_size > MAX_META_DATA)
    675         channel->mdata_size = MAX_META_DATA;
    676       if (NULL == (ndata = realloc (channel->mdata,
    677                                     channel->mdata_size)))
    678       {
    679         LOG_STRERROR ("realloc");
    680         channel->plugin->channel = NULL;
    681         channel->plugin->round_finished = 1;
    682         EXTRACTOR_IPC_channel_destroy_ (channel);
    683         channels[i] = NULL;
    684         continue;
    685       }
    686       channel->mdata = ndata;
    687     }
    688     do
    689     {
    690       iret = read (channel->cpipe_out,
    691                    &channel->mdata[channel->size],
    692                    channel->mdata_size - channel->size);
    693     }
    694     while ((-1 == iret) && (EINTR == errno));
    695     if ( (-1 == iret) ||
    696          (0 == iret) ||
    697          (-1 == (ret = EXTRACTOR_IPC_process_reply_ (channel->plugin,
    698                                                      channel->mdata,
    699                                                      channel->size + iret,
    700                                                      proc, proc_cls)) ) )
    701     {
    702       if (-1 == iret)
    703         LOG_STRERROR ("read");
    704       LOG ("Read error from channel, closing channel %s\n",
    705            channel->plugin->libname);
    706       /* The plugin is gone -- a crash in a plugin is exactly what this
    707          library is built to survive.  Forget the channel on the plugin
    708          as well, or the next file would be handed to a
    709          `struct EXTRACTOR_Channel' that has already been freed. */
    710       channel->plugin->channel = NULL;
    711       channel->plugin->round_finished = 1;
    712       EXTRACTOR_IPC_channel_destroy_ (channel);
    713       channels[i] = NULL;
    714       continue;
    715     }
    716     else
    717     {
    718       channel->started = 1;
    719       channel->size = channel->size + iret - ret;
    720       memmove (channel->mdata,
    721                &channel->mdata[ret],
    722                channel->size);
    723     }
    724   }
    725   return 1;
    726 }
    727 
    728 
    729 /* end of extractor_ipc_gnu.c */