libgpuverify

Signature verification on GPUs (WiP)
Log | Files | Refs | README | LICENSE

util.c (18601B)


      1 /*
      2  * util.c
      3  * This file is part of lib-gpu-verify.
      4  *
      5  * lib-gpu-verify is free software: you can redistribute it and/or modify
      6  * it under the terms of the GNU General Public License as published by
      7  * the Free Software Foundation, either version 3 of the License, or
      8  * (at your option) any later version.
      9  *
     10  * lib-gpu-verify is distributed in the hope that it will be useful,
     11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
     12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     13  * GNU General Public License for more details.
     14  *
     15  * Created by Cedric Zwahlen
     16  *
     17  */
     18 
     19 #include "util.h"
     20 
     21 unsigned long
     22 gpuvt_estimate_pairs (void)
     23 {
     24 
     25   struct stat ss;
     26 
     27   int msfile = open ("../lib-gpu-generate/msgsig.txt", O_RDONLY);
     28 
     29   if (-1 == msfile)
     30   {
     31     fprintf (stderr,
     32              "Failed to open: err %s\n",
     33              strerror (errno));
     34     exit (1);
     35   }
     36 
     37   fstat (msfile, &ss);
     38 
     39   unsigned long len_f = ss.st_size;
     40   unsigned long len_sig = (GPUV_BIT_LENGTH_2048 / 8) * 2 + 1;   // this is the size of a 2048 bit signature in the file
     41 
     42   unsigned long n_min = len_f / (len_sig + 3);   // if each message was only one character, then this would be the maximum amount of signatures that could be in the file – use this estimate to allocate storage for the signatures
     43 
     44   close (msfile);
     45 
     46   return n_min == 0 ? 1 : n_min;
     47 }
     48 
     49 
     50 void
     51 gpuv_prepare_gcry (void)
     52 {
     53 
     54 
     55   /* Version check should be the very first call because it
     56    makes sure that important subsystems are initialized.
     57    #define NEED_LIBGCRYPT_VERSION to the minimum required version. */
     58   if (! gcry_check_version (NEED_LIBGCRYPT_VERSION))
     59   {
     60     fprintf (stderr, "libgcrypt is too old (need %s, have %s)\n",
     61              NEED_LIBGCRYPT_VERSION, gcry_check_version (NULL));
     62     exit (2);
     63   }
     64   /* Disable secure memory.  */
     65   gcry_control (GCRYCTL_DISABLE_SECMEM, 0);
     66   /* ... If required, other initialization goes here.  */
     67   /* Tell Libgcrypt that initialization has completed. */
     68   gcry_control (GCRYCTL_INITIALIZATION_FINISHED, 0);
     69 
     70 }
     71 
     72 
     73 cl_platform_id
     74 select_platform (unsigned int offset,
     75                  bool print_platforms)
     76 {
     77   cl_uint max_platforms = 4;
     78   cl_platform_id platforms[max_platforms];
     79   cl_uint num_platforms;
     80   cl_int rplat;
     81 
     82   rplat = clGetPlatformIDs (max_platforms,
     83                             platforms,
     84                             &num_platforms);
     85   if (CL_SUCCESS != rplat)
     86   {
     87     fprintf (stderr,
     88              "Error: Failed to lookup platforms! Maybe install $VENDOR-opencl-icd package? (Error #%d)\n",
     89              rplat);
     90     exit (1);
     91   }
     92   if (print_platforms)
     93   {
     94     for (unsigned int i = 0; i<num_platforms; i++)
     95     {
     96       char buf[128];
     97       size_t rbuf;
     98       static struct
     99       {
    100         cl_platform_info cpi;
    101         const char *name;
    102       } param[] = {
    103         { CL_PLATFORM_PROFILE, "profile" },
    104         { CL_PLATFORM_VENDOR, "vendor" },
    105         { CL_PLATFORM_NAME, "name" },
    106         { CL_PLATFORM_EXTENSIONS, "extensions" },
    107         { 0, NULL }
    108       };
    109 
    110       for (unsigned int j = 0; NULL != param[j].name; j++)
    111       {
    112         cl_int err;
    113 
    114         err = clGetPlatformInfo (platforms[i],
    115                                  param[j].cpi,
    116                                  sizeof (buf),
    117                                  buf,
    118                                  &rbuf);
    119         if (err != CL_SUCCESS)
    120         {
    121           fprintf (stderr,
    122                    "Error: Failed to get platform info for %s! (%d)\n",
    123                    param[j].name,
    124                    err);
    125         }
    126         else
    127         {
    128           printf ("#%u %s %.*s\n",
    129                   i,
    130                   param[j].name,
    131                   (int) rbuf,
    132                   buf);
    133         }
    134       }
    135     }
    136     exit (0);
    137   }
    138   if (offset >= num_platforms)
    139   {
    140     fprintf (stderr,
    141              "Found only %u platforms\n",
    142              (unsigned int) num_platforms);
    143     exit (1);
    144   }
    145   return platforms[offset];
    146 }
    147 
    148 
    149 cl_device_id
    150 select_device (cl_platform_id platform)
    151 {
    152   cl_device_id device_id;
    153   char buf[1024];
    154   size_t len;
    155   cl_int err;
    156   cl_uint address_bits = 0;
    157 
    158   err = clGetDeviceIDs (platform,
    159                         CL_DEVICE_TYPE_ALL,
    160                         1, /* 1 device */
    161                         &device_id,
    162                         NULL);
    163   if (CL_SUCCESS != err)
    164   {
    165     fprintf (stderr,
    166              "Error: Failed to find a device! (%d)\n",
    167              err);
    168     exit (1);
    169   }
    170 
    171   err = clGetDeviceInfo (device_id,
    172                          CL_DRIVER_VERSION,
    173                          sizeof (buf),
    174                          buf,
    175                          &len);
    176   if (CL_SUCCESS != err)
    177   {
    178     fprintf (stderr,
    179              "Error: Failed to get device driver version! (%d)\n",
    180              err);
    181     exit (1);
    182   }
    183 //  printf ("Driver version: %.*s\n",
    184 //          (int) len,
    185 //          buf);
    186   clGetDeviceInfo (device_id,
    187                    CL_DEVICE_ADDRESS_BITS,
    188                    sizeof (address_bits),
    189                    &address_bits,
    190                    &len);
    191   if (CL_SUCCESS != err)
    192   {
    193     fprintf (stderr,
    194              "Error: Failed to get device address bits! (%d)\n",
    195              err);
    196     exit (1);
    197   }
    198 //  printf ("device address bits: %d\n",
    199 //          (int) address_bits);
    200   return device_id;
    201 }
    202 
    203 
    204 void
    205 logger (const char *errinfo,
    206         const void *private_info,
    207         size_t cb,
    208         void *user_data)
    209 {
    210   fprintf (stderr,
    211            "<OpenCL>: %s\n",
    212            errinfo);
    213 }
    214 
    215 
    216 cl_context
    217 create_compute_context (cl_device_id device_id)
    218 {
    219   cl_int err;
    220   cl_context context;
    221 
    222   context = clCreateContext (NULL,
    223                              1,
    224                              &device_id,
    225                              &logger, NULL,
    226                              &err);
    227   if (CL_SUCCESS != err)
    228   {
    229     fprintf (stderr,
    230              "Error: Failed to create a compute context (%d)\n",
    231              err);
    232     exit (1);
    233   }
    234   return context;
    235 }
    236 
    237 
    238 cl_command_queue
    239 create_command_queue (cl_device_id device_id,
    240                       cl_context context)
    241 {
    242   cl_int err;
    243   cl_command_queue commands;
    244 
    245   commands = clCreateCommandQueue (context,
    246                                    device_id,
    247                                    0, /* properties */
    248                                    &err);
    249   if (CL_SUCCESS != err)
    250   {
    251     fprintf (stderr,
    252              "Error: Failed to create a command queue!\n (%d)",
    253              err);
    254     exit (1);
    255   }
    256   return commands;
    257 }
    258 
    259 
    260 cl_program
    261 compile_program (cl_device_id device_id,
    262                  cl_context context,
    263                  const char *sourcefile)
    264 {
    265   cl_program program;
    266   int fd;
    267   void *code;
    268   struct stat ss;
    269   cl_int err;
    270 
    271   fd = open (sourcefile,
    272              O_RDONLY);
    273   if (-1 == fd)
    274   {
    275     fprintf (stderr,
    276              "Failed to open %s: %s\n",
    277              sourcefile,
    278              strerror (errno));
    279     exit (1);
    280   }
    281   if (0 != fstat (fd,
    282                   &ss))
    283   {
    284     fprintf (stderr,
    285              "Failed to stat %s: %s\n",
    286              sourcefile,
    287              strerror (errno));
    288     close (fd);
    289     exit (1);
    290   }
    291   code = mmap (NULL,
    292                ss.st_size,
    293                PROT_READ,
    294                MAP_PRIVATE,
    295                fd,
    296                0 /* offset */);
    297   close (fd);
    298   {
    299     size_t sz = ss.st_size;
    300 
    301     program = clCreateProgramWithSource (context,
    302                                          1, /* 1 source file */
    303                                          (const char **) &code,
    304                                          &sz,
    305                                          &err);
    306     if (CL_SUCCESS != err)
    307     {
    308       fprintf (stderr,
    309                "Error: Failed to create compute program (%d)!\n",
    310                err);
    311       munmap (code,
    312               ss.st_size);
    313       exit (1);
    314     }
    315   }
    316   err = clBuildProgram (program,
    317                         0, /* number of devices */
    318                         NULL, /* devices */
    319                         NULL, /* options (char *) */
    320                         NULL, /* callback */
    321                         NULL);
    322   munmap (code,
    323           ss.st_size);
    324   if (CL_SUCCESS != err)
    325   {
    326     size_t len;
    327     char buffer[2048];
    328 
    329     fprintf (stderr,
    330              "Error: Failed to build program executable (%d)!\n",
    331              err);
    332     err = clGetProgramBuildInfo (program,
    333                                  device_id,
    334                                  CL_PROGRAM_BUILD_LOG,
    335                                  sizeof(buffer),
    336                                  buffer,
    337                                  &len);
    338     if (CL_SUCCESS != err)
    339     {
    340       fprintf (stderr,
    341                "Error: could not get build logs (%d)!\n",
    342                err);
    343       exit (1);
    344     }
    345     fprintf (stderr,
    346              "<clBuild>: %.*s\n",
    347              (int) len,
    348              buffer);
    349     exit (1);
    350   }
    351   return program;
    352 }
    353 
    354 
    355 cl_kernel
    356 create_kernel (cl_program program,
    357                const char *name)
    358 {
    359   cl_kernel kernel;
    360   cl_int err;
    361 
    362   kernel = clCreateKernel (program,
    363                            name,
    364                            &err);
    365   if (CL_SUCCESS != err)
    366   {
    367     fprintf (stderr,
    368              "Error: Failed to create compute kernel %s: %d!\n",
    369              name,
    370              err);
    371     exit (1);
    372   }
    373   return kernel;
    374 }
    375 
    376 
    377 // implementations of functions used by universal, but they themselves should not be exposed
    378 
    379 /*
    380  called after results has been copied from the gpu. releases many of the state variables, and invalidates the object
    381  calls the users closure
    382  */
    383 void CL_CALLBACK
    384 callback_result (cl_event event, cl_int event_command_status, void *user_data)
    385 {
    386 
    387   // clReleaseEvent(event);
    388 
    389   struct gpuv_state *state = (struct gpuv_state *) user_data;
    390 
    391   // finish calculating results, and release memory
    392 
    393   clReleaseMemObject (state->x_mem);
    394   clReleaseMemObject (state->m_mem);
    395   clReleaseMemObject (state->n_mem);
    396   clReleaseMemObject (state->ni_mem);
    397   clReleaseMemObject (state->exp_mem);
    398   clReleaseMemObject (state->msg_mem);
    399   clReleaseMemObject (state->pks_indices);
    400 
    401   clReleaseCommandQueue (state->queue);
    402 
    403   state->stale = 1;
    404 
    405   struct timespec p1 = state->t;
    406 
    407   clock_gettime (CLOCK_REALTIME, &state->t);
    408 
    409   state->t.tv_sec = (state->t.tv_nsec < p1.tv_nsec ? state->t.tv_sec
    410                      - (p1.tv_sec + 1) : state->t.tv_sec - p1.tv_sec);
    411   state->t.tv_nsec = (state->t.tv_nsec < p1.tv_nsec ? ((999999999
    412                                                         - p1.tv_nsec)
    413                                                        + state->t.tv_nsec) :
    414                       (state->t.tv_nsec - p1.tv_nsec) ) / 1000;
    415 
    416   // the user could check this themselves, but I give them a boolean field valid, that says whether there is a faulty signature in the batch or not
    417 
    418   int ret = 1;
    419 
    420   unsigned long partial = state->sig_count / (sizeof(uint32_t) * 8);
    421 
    422   for (int i = 0; i < state->results_len; i++)
    423   {
    424 
    425     uint32_t mask = 0;
    426 
    427     if (i >= partial)
    428     {
    429       int remaining = state->sig_count % (sizeof(uint32_t) * 8);
    430 
    431       for (int x = 0; x < remaining; x++)
    432       {
    433         mask |= 1 << x;
    434       }
    435 
    436     }
    437     else
    438     {
    439       mask = UINT32_MAX;
    440     }
    441 
    442     if (state->results[i] != mask)
    443     {
    444       ret = 0;
    445     }
    446   }
    447 
    448   state->info->in_progress = 0;
    449   state->valid = ret;
    450 
    451   // pass results to user
    452   state->cls (state->arg, state->valid, state->t,  state->results_len,
    453               state->results);
    454 
    455 }
    456 
    457 
    458 /*
    459  called after kernel finishes. launches another async process to copy the results from gpu memory
    460  */
    461 void CL_CALLBACK
    462 callback_kernel (cl_event event, cl_int event_command_status, void *user_data)
    463 {
    464 
    465   // clReleaseEvent(event); // only call here if we wait for completion
    466 
    467   struct gpuv_state *state = (struct gpuv_state *) user_data;
    468 
    469   // MARK: no expensive operations here
    470 
    471   unsigned long res_len = state->results_len;
    472   unsigned long res_len_bytes = res_len * sizeof(uint32_t);
    473 
    474   int err = 0;
    475 
    476   cl_event e = clCreateUserEvent (state->info->context, NULL);
    477 
    478   // Read back the results from the device to verify the output
    479   err = clEnqueueReadBuffer (state->queue, state->res_mem, CL_TRUE, 0,
    480                              res_len_bytes, state->results, 0, NULL, &e);
    481   if (err != CL_SUCCESS)
    482   {
    483     printf ("Error: Failed to read output array! %d\n", err);
    484     exit (1);
    485   }
    486 
    487   clSetEventCallback (e, CL_COMPLETE, callback_result, state);
    488   clReleaseEvent (e);
    489 }
    490 
    491 
    492 #define ORDER -1 // I think we need to do this, because we want to write it in the 'wrong' way
    493 #define END 0
    494 
    495 // #define GPUV_BIT_LENGTH 2048
    496 
    497 #define BITS 64
    498 
    499 void
    500 pk_to_mont (struct gpuv_public_key *pk)
    501 {
    502 
    503   if (pk->prepared)
    504     return;
    505 
    506   gpu_register *ni_buf = malloc (pk->len_n * sizeof(gpu_register));
    507   memset (ni_buf, 0, pk->len_n * sizeof(gpu_register));
    508   gpu_register *r_1_buf = malloc (pk->len_n * sizeof(gpu_register));
    509   memset (r_1_buf, 0, pk->len_n * sizeof(gpu_register));
    510 
    511   mpz_t mod, r, r_1, ni;
    512 
    513   mpz_init (r);
    514   mpz_init (r_1);
    515   mpz_init (ni);
    516   mpz_init (mod);
    517 
    518   mpz_t one;   // helper variable
    519   mpz_init_set_si (one,1);
    520 
    521   mpz_set_si (one, 1);
    522   mpz_mul_2exp (r,one,GPUV_BIT_LENGTH_2048);  // r
    523 
    524   mpz_import (mod, pk->len_n, ORDER, sizeof(gpu_register), END, 0, pk->n);
    525 
    526   mpz_gcdext (one, r_1, ni, r, mod);  // set r_1 and ni
    527 
    528   int sgn = mpz_sgn (r_1);
    529 
    530   mpz_abs (r_1, r_1);
    531   mpz_abs (ni, ni);
    532 
    533   if (sgn == -1)
    534   {
    535     mpz_sub (ni, r, ni);
    536     mpz_sub (r_1, mod, r_1);
    537   }
    538 
    539   mpz_export (ni_buf, NULL, ORDER, sizeof(gpu_register), END, 0, ni);
    540   mpz_export (r_1_buf, NULL, ORDER, sizeof(gpu_register), END, 0, r_1);
    541 
    542   pk->ni = (char *) ni_buf;
    543   pk->len_ni = pk->len_n;   // has to be the same length as n
    544   pk->r_1 = (char *) r_1_buf;
    545   pk->len_r_1 = pk->len_n;
    546 
    547   mpz_clear (r);
    548   mpz_clear (r_1);
    549   mpz_clear (ni);
    550   mpz_clear (one);
    551   mpz_clear (mod);
    552 
    553   pk->prepared = 1;
    554 
    555 }
    556 
    557 
    558 void
    559 sig_msg_to_mont (struct gpuv_signature_message *sig_msg)
    560 {
    561 
    562   if (sig_msg->prepared)
    563   {
    564 
    565     return;
    566   }
    567 
    568   gpu_register *x_buf = malloc (sig_msg->pubkey->len_n * sizeof(gpu_register));
    569   memset (x_buf, 0, sig_msg->pubkey->len_n * sizeof(gpu_register));
    570   gpu_register *M_buf = malloc (sig_msg->pubkey->len_n * sizeof(gpu_register));
    571   memset (M_buf, 0, sig_msg->pubkey->len_n * sizeof(gpu_register));
    572 
    573 
    574   mpz_t mod, sig;
    575   mpz_init (sig);
    576   mpz_init (mod);
    577 
    578   mpz_t M, x, r;
    579 
    580   mpz_init (M);
    581   mpz_init (x);
    582   mpz_init (r);
    583 
    584   mpz_t one;   // helper variable
    585   mpz_init_set_si (one,1);
    586 
    587   mpz_set_si (one, 1);
    588   mpz_mul_2exp (r,one,GPUV_BIT_LENGTH_2048);  // r
    589 
    590   mpz_import (mod, sig_msg->pubkey->len_n, ORDER, sizeof(gpu_register), END, 0,
    591               sig_msg->pubkey->n);
    592 
    593   mpz_import (sig, sig_msg->len_s, ORDER, sizeof(gpu_register), END, 0,
    594               sig_msg->s);
    595 
    596   // set x (the number to 'square' (multiply by itself))
    597   mpz_mul (M, sig, r);
    598   mpz_mod (M, M, mod);
    599 
    600   mpz_mod (x, r, mod);
    601 
    602   mpz_export (x_buf, NULL, ORDER, sizeof(gpu_register), END, 0, x);
    603   mpz_export (M_buf, NULL, ORDER, sizeof(gpu_register), END, 0, M);
    604 
    605   mpz_clear (sig);
    606   mpz_clear (mod);
    607   mpz_clear (r);
    608   mpz_clear (one);
    609   mpz_clear (M);
    610   mpz_clear (x);
    611 
    612   sig_msg->x = (char *) x_buf;
    613   sig_msg->len_x = sig_msg->pubkey->len_n;
    614   sig_msg->M = (char *) M_buf;
    615   sig_msg->len_M = sig_msg->pubkey->len_n;
    616 
    617   sig_msg->prepared = 1;
    618 
    619 }
    620 
    621 
    622 gcry_sexp_t *
    623 sexp_from_string (char*str, const char *format)
    624 {
    625 
    626   gcry_sexp_t *sexp = malloc (sizeof(gcry_sexp_t));
    627 
    628   gcry_mpi_t mpi;
    629 
    630   char *str_buf = malloc ((GPUV_BIT_LENGTH_2048 / 8) * 2 + 8);
    631 
    632   mpz_t m;
    633   mpz_init (m);
    634 
    635   mpz_import (m, 32, ORDER, sizeof(gpu_register), END, 0, str);
    636 
    637   mpz_get_str (str_buf, 16, m);
    638 
    639 
    640   gcry_mpi_scan (&mpi, GCRYMPI_FMT_HEX, str_buf, 0, NULL);
    641 
    642   size_t errOff = 0;
    643   gcry_sexp_build (sexp,&errOff,format,mpi);
    644 
    645   gcry_mpi_release (mpi);
    646   free (str_buf);
    647 
    648   mpz_clear (m);
    649 
    650   return sexp;
    651 }
    652 
    653 
    654 gcry_sexp_t *
    655 sexp_from_string_key (char*str_1, char*str_1_buf, unsigned long str_2, const
    656                       char *format)
    657 {
    658 
    659   gcry_sexp_t *sexp = malloc (sizeof(gcry_sexp_t));
    660 
    661   gcry_mpi_t mpi_1;
    662 
    663 
    664   mpz_t m;
    665   mpz_init (m);
    666 
    667   mpz_import (m, 32, ORDER, sizeof(gpu_register), END, 0, str_1);
    668 
    669   mpz_get_str (str_1_buf, 16, m);
    670 
    671   gcry_mpi_scan (&mpi_1, GCRYMPI_FMT_HEX, str_1_buf, 0, NULL);
    672 
    673   gcry_mpi_t mpi_2 = gcry_mpi_set_ui (NULL, str_2);
    674 
    675   size_t errOff = 0;
    676   gcry_sexp_build (sexp,&errOff,format,mpi_1,mpi_2);
    677 
    678   gcry_mpi_release (mpi_1);
    679   gcry_mpi_release (mpi_2);
    680 
    681   mpz_clear (m);
    682 
    683   return sexp;
    684 }
    685 
    686 
    687 void
    688 cpu_verify (struct gpuv_batch *batch, struct gpuv_state *state)
    689 {
    690 
    691   struct timespec p1, p2;
    692 
    693   clock_gettime (CLOCK_REALTIME, &p1);
    694 
    695   for (int j = 0; j < batch->current; j++)
    696   {
    697 
    698     struct gpuv_public_key *p = batch->pairs[ j ].pubkey;
    699 
    700     char *str_key_buf = malloc ((GPUV_BIT_LENGTH_2048 / 8) * 2 + 8);
    701 
    702     gcry_sexp_t *key_sexp = sexp_from_string_key (p->n, str_key_buf, p->e,
    703                                                   "(public-key (rsa (n %m) (e %m)))");                               // pub key data
    704 
    705     gcry_sexp_t *m_sexp = sexp_from_string (batch->pairs[j].m,
    706                                             "(data (flags raw) (value %m))");                        // message format (for comparison)
    707 
    708     gcry_sexp_t *s_sexp = sexp_from_string (batch->pairs[j].s,
    709                                             "(sig-val (rsa (s %m)))");                       // signature format
    710 
    711     if (gcry_pk_verify (*s_sexp, *m_sexp, *key_sexp) == 0)
    712     {
    713 
    714       uint32_t out_offset = j / (sizeof(uint32_t) * 8);       // 32 bit
    715 
    716       uint32_t mv = 1 << j;
    717 
    718       state->results[out_offset] |= mv;
    719 
    720     }
    721 
    722     gcry_sexp_release (*m_sexp);
    723     gcry_sexp_release (*s_sexp);
    724     gcry_sexp_release (*key_sexp);
    725 
    726     free (str_key_buf);
    727 
    728   }
    729 
    730   state->stale = 1;
    731 
    732   int ret = 1;
    733 
    734   unsigned long partial = state->sig_count / (sizeof(uint32_t) * 8);
    735 
    736   for (int i = 0; i < state->results_len; i++)
    737   {
    738 
    739     uint32_t mask = 0;
    740 
    741     if (i >= partial)
    742     {
    743       int remaining = state->sig_count % (sizeof(uint32_t) * 8);
    744 
    745       for (int x = 0; x < remaining; x++)
    746       {
    747         mask |= 1 << x;
    748       }
    749 
    750     }
    751     else
    752     {
    753       mask = UINT32_MAX;
    754     }
    755 
    756     if (state->results[i] != mask)
    757     {
    758       ret = 0;
    759     }
    760   }
    761 
    762   state->info->in_progress = 0;
    763   state->valid = ret;
    764 
    765   clock_gettime (CLOCK_REALTIME, &p2);
    766 
    767   state->t.tv_sec = (p2.tv_nsec < p1.tv_nsec ? p2.tv_sec - (p1.tv_sec + 1) :
    768                      p2.tv_sec - p1.tv_sec);
    769   state->t.tv_nsec = (p2.tv_nsec < p1.tv_nsec ? ((999999999 - p1.tv_nsec)
    770                                                  + p2.tv_nsec) : (p2.tv_nsec
    771                                                                   - p1.tv_nsec) )
    772                      / 1000;
    773 
    774   // pass results to user
    775   state->cls (state->arg, state->valid, state->t,  state->results_len,
    776               state->results);
    777 
    778 }