1  /***********************************************************************
  2   * Distributed under the MIT software license, see the accompanying    *
  3   * file COPYING or https://www.opensource.org/licenses/mit-license.php.*
  4   ***********************************************************************/
  5  
  6  #include <stdio.h>
  7  #include <stdlib.h>
  8  #include <string.h>
  9  
 10  #if defined(SUPPORTS_CONCURRENCY)
 11  #include <sys/types.h>
 12  #include <sys/wait.h>
 13  #include <unistd.h>
 14  #endif
 15  
 16  #include "unit_test.h"
 17  #include "testrand.h"
 18  #include "tests_common.h"
 19  
 20  #define UNUSED(x) (void)(x)
 21  
 22  /* Number of times certain tests will run */
 23  int COUNT = 16;
 24  
 25  static int parse_jobs_count(const char* key, const char* value, struct tf_framework* tf);
 26  static int parse_iterations(const char* key, const char* value, struct tf_framework* tf);
 27  static int parse_seed(const char* key, const char* value, struct tf_framework* tf);
 28  static int parse_target(const char* key, const char* value, struct tf_framework* tf);
 29  static int parse_logging(const char* key, const char* value, struct tf_framework* tf);
 30  
 31  /* Mapping table: key -> handler */
 32  typedef int (*ArgHandler)(const char* key, const char* value, struct tf_framework* tf);
 33  struct ArgMap {
 34      const char* key;
 35      ArgHandler handler;
 36  };
 37  
 38  /*
 39   *   Main entry point for handling command-line arguments.
 40   *
 41   *   Developers should extend this map whenever new command-line
 42   *   options are introduced. Each new argument should be validated,
 43   *   converted to the appropriate type, and stored in 'tf->args' struct.
 44   */
 45  static struct ArgMap arg_map[] = {
 46      { "t", parse_target }, { "target", parse_target },
 47      { "j", parse_jobs_count }, { "jobs", parse_jobs_count },
 48      { "i", parse_iterations }, { "iterations", parse_iterations },
 49      { "seed", parse_seed },
 50      { "log", parse_logging },
 51      { NULL, NULL } /* sentinel */
 52  };
 53  
 54  /* Display options that are not printed elsewhere */
 55  static void print_args(const struct tf_args* args) {
 56      printf("iterations = %d\n", COUNT);
 57      printf("jobs = %d. %s execution.\n", args->num_processes, args->num_processes > 1 ? "Parallel" : "Sequential");
 58  }
 59  
 60  /* Main entry point for reading environment variables */
 61  static int read_env(struct tf_framework* tf) {
 62      const char* env_iter = getenv("SECP256K1_TEST_ITERS");
 63      if (env_iter && strlen(env_iter) > 0) {
 64          return parse_iterations("i", env_iter, tf);
 65      }
 66      return 0;
 67  }
 68  
 69  static int parse_arg(const char* key, const char* value, struct tf_framework* tf) {
 70      int i;
 71      for (i = 0; arg_map[i].key != NULL; i++) {
 72          if (strcmp(key, arg_map[i].key) == 0) {
 73              return arg_map[i].handler(key, value, tf);
 74          }
 75      }
 76      /* Unknown key: report just so typos don't silently pass. */
 77      fprintf(stderr, "Unknown argument '-%s=%s'\n", key, value);
 78      return -1;
 79  }
 80  
 81  static void help(void) {
 82      printf("Usage: ./tests [options]\n\n");
 83      printf("Run the test suite for the project with optional configuration.\n\n");
 84      printf("Options:\n");
 85      printf("    --help, -h                           Show this help message\n");
 86      printf("    --list_tests, -l                     Display list of all available tests and modules\n");
 87      printf("    --jobs=<num>, -j=<num>               Number of parallel worker processes (default: 0 = sequential)\n");
 88      printf("    --iterations=<num>, -i=<num>         Number of iterations for each test (default: 16)\n");
 89      printf("    --seed=<hex>                         Set a specific RNG seed (default: random)\n");
 90      printf("    --target=<test name>, -t=<name>      Run a specific test (can be provided multiple times)\n");
 91      printf("    --target=<module name>, -t=<module>  Run all tests within a specific module (can be provided multiple times)\n");
 92      printf("    --log=<0|1>                          Enable or disable test execution logging (default: 0 = disabled)\n");
 93      printf("\n");
 94      printf("Notes:\n");
 95      printf("    - All arguments must be provided in the form '--key=value', '-key=value' or '-k=value'.\n");
 96      printf("    - Single or double dashes are allowed for multi character options.\n");
 97      printf("    - Unknown arguments are reported but ignored.\n");
 98      printf("    - Sequential execution occurs if -jobs=0 or unspecified.\n");
 99      printf("    - Iterations and seed can also be passed as positional arguments before any other argument for backward compatibility.\n");
100  }
101  
102  /* Print all tests in registry */
103  static void print_test_list(struct tf_framework* tf) {
104      int m, t, total = 0;
105      printf("\nAvailable tests (%d modules):\n", tf->num_modules);
106      printf("========================================\n");
107      for (m = 0; m < tf->num_modules; m++) {
108          const struct tf_test_module* mod = &tf->registry_modules[m];
109          printf("Module: %s (%d tests)\n", mod->name, mod->size);
110          for (t = 0; t < mod->size; t++) {
111              printf("\t[%3d] %s\n", total + 1, mod->data[t].name);
112              total++;
113          }
114          printf("----------------------------------------\n");
115      }
116      printf("\nRun specific module: ./tests -t=<module_name>\n");
117      printf("Run specific test: ./tests -t=<test_name>\n\n");
118  }
119  
120  static int parse_jobs_count(const char* key, const char* value, struct tf_framework* tf) {
121      char* ptr_val;
122      long val = strtol(value, &ptr_val, 10); /* base 10 */
123      if (*ptr_val != '\0') {
124          fprintf(stderr, "Invalid number for -%s=%s\n", key, value);
125          return -1;
126      }
127      if (val < 0 || val > MAX_SUBPROCESSES) {
128          fprintf(stderr, "Arg '-%s' out of range: '%ld'. Range: 0..%d\n", key, val, MAX_SUBPROCESSES);
129          return -1;
130      }
131      tf->args.num_processes = (int) val;
132      return 0;
133  }
134  
135  static int parse_iterations(const char* key, const char* value, struct tf_framework* tf) {
136      UNUSED(key); UNUSED(tf);
137      if (!value) return 0;
138      COUNT = (int) strtol(value, NULL, 0);
139      if (COUNT <= 0) {
140          fputs("An iteration count of 0 or less is not allowed.\n", stderr);
141          return -1;
142      }
143      return 0;
144  }
145  
146  static int parse_seed(const char* key, const char* value, struct tf_framework* tf) {
147      UNUSED(key);
148      tf->args.custom_seed = (!value || strcmp(value, "NULL") == 0) ? NULL : value;
149      return 0;
150  }
151  
152  static int parse_logging(const char* key, const char* value, struct tf_framework* tf) {
153      UNUSED(key);
154      tf->args.logging = value && strcmp(value, "1") == 0;
155      return 0;
156  }
157  
158  /* Strip up to two leading dashes */
159  static const char* normalize_key(const char* arg, const char** err_msg) {
160      const char* key;
161      if (!arg || arg[0] != '-') {
162          *err_msg = "missing initial dash";
163          return NULL;
164      }
165      /* single-dash short option */
166      if (arg[1] != '-') return arg + 1;
167  
168      /* double-dash checks now */
169      if (arg[2] == '\0') {
170          *err_msg = "missing option name after double dash";
171          return NULL;
172      }
173  
174      if (arg[2] == '-') {
175          *err_msg = "too many leading dashes";
176          return NULL;
177      }
178  
179      key = arg + 2;
180      if (key[1] == '\0') {
181          *err_msg = "short option cannot use double dash";
182          return NULL;
183      }
184      return key;
185  }
186  
187  static int parse_target(const char* key, const char* value, struct tf_framework* tf) {
188      int group, idx;
189      const struct tf_test_entry* entry;
190      UNUSED(key);
191      /* Find test index in the registry */
192      for (group = 0; group < tf->num_modules; group++) {
193          const struct tf_test_module* module = &tf->registry_modules[group];
194          int add_all = strcmp(value, module->name) == 0; /* select all from module */
195          for (idx = 0; idx < module->size; idx++) {
196              entry = &module->data[idx];
197              if (add_all || strcmp(value, entry->name) == 0) {
198                  if (tf->args.targets.size >= MAX_ARGS) {
199                      fprintf(stderr, "Too many -target args (max: %d)\n", MAX_ARGS);
200                      return -1;
201                  }
202                  tf->args.targets.slots[tf->args.targets.size++] = entry;
203                  /* Matched a single test, we're done */
204                  if (!add_all) return 0;
205              }
206          }
207          /* If add_all was true, we added all tests in the module, so return */
208          if (add_all) return 0;
209      }
210      fprintf(stderr, "Error: target '%s' not found (missing or module disabled).\n"
211                      "Run program with -list_tests option to display available tests and modules.\n", value);
212      return -1;
213  }
214  
215  /* Read args: all must be in the form -key=value, --key=value or -key=value */
216  static int read_args(int argc, char** argv, int start, struct tf_framework* tf) {
217      int i;
218      const char* key;
219      const char* value;
220      char* eq;
221      const char* err_msg = "unknown error";
222      for (i = start; i < argc; i++) {
223          char* raw_arg = argv[i];
224          if (!raw_arg || raw_arg[0] != '-') {
225              fprintf(stderr, "Invalid arg '%s': must start with '-'\n", raw_arg ? raw_arg : "(null)");
226              return -1;
227          }
228  
229          key = normalize_key(raw_arg, &err_msg);
230          if (!key || *key == '\0') {
231              fprintf(stderr, "Invalid arg '%s': %s. Must be -k=value or --key=value\n", raw_arg, err_msg);
232              return -1;
233          }
234  
235          eq = strchr(raw_arg, '=');
236          if (!eq || eq == raw_arg + 1) {
237              /* Allowed options without value */
238              if (strcmp(key, "h") == 0 || strcmp(key, "help") == 0) {
239                  tf->args.help = 1;
240                  return 0;
241              }
242              if (strcmp(key, "l") == 0 || strcmp(key, "list_tests") == 0) {
243                  tf->args.list_tests = 1;
244                  return 0;
245              }
246              fprintf(stderr, "Invalid arg '%s': must be -k=value or --key=value\n", raw_arg);
247              return -1;
248          }
249  
250          *eq = '\0'; /* split key and value */
251          value = eq + 1;
252          if (!value || *value == '\0') { /* value is empty */
253              fprintf(stderr, "Invalid arg '%s': value cannot be empty\n", raw_arg);
254              return -1;
255          }
256  
257          if (parse_arg(key, value, tf) != 0) return -1;
258      }
259      return 0;
260  }
261  
262  static void run_test_log(const struct tf_test_entry* t) {
263      int64_t start_time = gettime_i64();
264      printf("Running %s..\n", t->name);
265      t->func();
266      printf("Test %s PASSED (%.3f sec)\n", t->name, (double)(gettime_i64() - start_time) / 1000000);
267  }
268  
269  static void run_test(const struct tf_test_entry* t) { t->func(); }
270  
271  /* Process tests in sequential order */
272  static int run_sequential(struct tf_framework* tf) {
273      int it;
274      for (it = 0; it < tf->args.targets.size; it++) {
275          tf->fn_run_test(tf->args.targets.slots[it]);
276      }
277      return EXIT_SUCCESS;
278  }
279  
280  #if defined(SUPPORTS_CONCURRENCY)
281  static const int MAX_TARGETS = 255;
282  
283  /* Process tests in parallel */
284  static int run_concurrent(struct tf_framework* tf) {
285      /* Sub-processes info */
286      pid_t workers[MAX_SUBPROCESSES];
287      int pipefd[2];
288      int status = EXIT_SUCCESS;
289      int it; /* loop iterator */
290      unsigned char idx; /* test index */
291  
292      if (tf->args.targets.size > MAX_TARGETS) {
293          fprintf(stderr, "Internal Error: the number of targets (%d) exceeds the maximum supported (%d). "
294                  "If you need more, extend 'run_concurrent()' to handle additional targets.\n",
295                  tf->args.targets.size, MAX_TARGETS);
296          exit(EXIT_FAILURE);
297      }
298  
299  
300      if (pipe(pipefd) != 0) {
301          perror("Error during pipe setup");
302          return EXIT_FAILURE;
303      }
304  
305      /* Launch worker processes */
306      for (it = 0; it < tf->args.num_processes; it++) {
307          pid_t pid = fork();
308          if (pid < 0) {
309              perror("Error during process fork");
310              return EXIT_FAILURE;
311          }
312          if (pid == 0) {
313              /* Child worker: read jobs from the shared pipe */
314              close(pipefd[1]); /* children never write */
315              while (read(pipefd[0], &idx, sizeof(idx)) == sizeof(idx)) {
316                  tf->fn_run_test(tf->args.targets.slots[(int)idx]);
317              }
318              _exit(EXIT_SUCCESS); /* finish child process */
319          } else {
320              /* Parent: save worker pid */
321              workers[it] = pid;
322          }
323      }
324  
325      /* Parent: write all tasks into the pipe */
326      close(pipefd[0]); /* close read end */
327      for (it = 0; it < tf->args.targets.size; it++) {
328          idx = (unsigned char)it;
329          if (write(pipefd[1], &idx, sizeof(idx)) == -1) {
330              perror("Error during workload distribution");
331              close(pipefd[1]);
332              return EXIT_FAILURE;
333          }
334      }
335      /* Close write end to signal EOF */
336      close(pipefd[1]);
337      /* Wait for all workers */
338      for (it = 0; it < tf->args.num_processes; it++) {
339          int ret = 0;
340          if (waitpid(workers[it], &ret, 0) == -1 || ret != 0) {
341              status = EXIT_FAILURE;
342          }
343      }
344  
345      return status;
346  }
347  #endif
348  
349  static int tf_init(struct tf_framework* tf, int argc, char** argv)
350  {
351      /* Caller must set the registry and its size before calling tf_init */
352      if (tf->registry_modules == NULL || tf->num_modules <= 0) {
353          fprintf(stderr, "Error: tests registry not provided or empty\n");
354          return EXIT_FAILURE;
355      }
356  
357      /* Initialize command-line options */
358      tf->args.num_processes = 0;
359      tf->args.custom_seed = NULL;
360      tf->args.help = 0;
361      tf->args.targets.size = 0;
362      tf->args.list_tests = 0;
363      tf->args.logging = 0;
364  
365      /* Disable buffering for stdout to improve reliability of getting
366       * diagnostic information. Happens right at the start of main because
367       * setbuf must be used before any other operation on the stream. */
368      setbuf(stdout, NULL);
369      /* Also disable buffering for stderr because it's not guaranteed that it's
370       * unbuffered on all systems. */
371      setbuf(stderr, NULL);
372  
373      /* Parse env args */
374      if (read_env(tf) != 0) return EXIT_FAILURE;
375  
376      /* Parse command-line args */
377      if (argc > 1) {
378          int named_arg_start = 1; /* index to begin processing named arguments */
379          if (argc - 1 > MAX_ARGS) { /* first arg is always the binary path */
380              fprintf(stderr, "Too many command-line arguments (max: %d)\n", MAX_ARGS);
381              return EXIT_FAILURE;
382          }
383  
384          /* Compatibility Note: The first two args were the number of iterations and the seed. */
385          /* If provided, parse them and adjust the starting index for named arguments accordingly. */
386          if (argv[1][0] != '-') {
387              int has_seed = argc > 2 && argv[2][0] != '-';
388              if (parse_iterations("i", argv[1], tf) != 0) return EXIT_FAILURE;
389              if (has_seed) parse_seed("seed", argv[2], tf);
390              named_arg_start = has_seed ? 3 : 2;
391          }
392          if (read_args(argc, argv, named_arg_start, tf) != 0) {
393              return EXIT_FAILURE;
394          }
395  
396          if (tf->args.help) {
397              help();
398              exit(EXIT_SUCCESS);
399          }
400  
401          if (tf->args.list_tests) {
402              print_test_list(tf);
403              exit(EXIT_SUCCESS);
404          }
405      }
406  
407      tf->fn_run_test = tf->args.logging ? run_test_log : run_test;
408      return EXIT_SUCCESS;
409  }
410  
411  static int tf_run(struct tf_framework* tf) {
412      /* Process exit status */
413      int status;
414      /* Whether to run all tests */
415      int run_all;
416      /* Loop iterator */
417      int it;
418      /* Initial test time */
419      int64_t start_time = gettime_i64();
420      /* Verify 'tf_init' has been called */
421      if (!tf->fn_run_test) {
422          fprintf(stderr, "Error: No test runner set. You must call 'tf_init' first to initialize the framework "
423                          "or manually assign 'fn_run_test' before calling 'tf_run'.\n");
424          return EXIT_FAILURE;
425      }
426  
427      /* Populate targets with all tests if none were explicitly specified */
428      run_all = tf->args.targets.size == 0;
429      if (run_all) {
430          int group, idx;
431          for (group = 0; group < tf->num_modules; group++) {
432              const struct tf_test_module* module = &tf->registry_modules[group];
433              for (idx = 0; idx < module->size; idx++) {
434                  if (tf->args.targets.size >= MAX_ARGS) {
435                      fprintf(stderr, "Internal Error: Number of tests (%d) exceeds MAX_ARGS (%d). "
436                                      "Increase MAX_ARGS to accommodate all tests.\n", tf->args.targets.size, MAX_ARGS);
437                      return EXIT_FAILURE;
438                  }
439                  tf->args.targets.slots[tf->args.targets.size++] = &module->data[idx];
440              }
441          }
442      }
443  
444      if (!tf->args.logging) printf("Tests running silently. Use '-log=1' to enable detailed logging\n");
445  
446      /* Log configuration */
447      print_args(&tf->args);
448  
449      /* Run test RNG tests (must run before we really initialize the test RNG) */
450      /* Note: currently, these tests are executed sequentially because there */
451      /* is really only one test. */
452      for (it = 0; tf->registry_no_rng && it < tf->registry_no_rng->size; it++) {
453          if (run_all) { /* future: support filtering */
454              tf->fn_run_test(&tf->registry_no_rng->data[it]);
455          }
456      }
457  
458      /* Initialize test RNG and library contexts */
459      testrand_init(tf->args.custom_seed);
460      if (tf->fn_setup && tf->fn_setup() != 0) return EXIT_FAILURE;
461  
462      /* Check whether to process tests sequentially or concurrently */
463      if (tf->args.num_processes <= 1) {
464          status = run_sequential(tf);
465      } else {
466  #if defined(SUPPORTS_CONCURRENCY)
467          status = run_concurrent(tf);
468  #else
469          fputs("Parallel execution not supported on your system. Running sequentially...\n", stderr);
470          status = run_sequential(tf);
471  #endif
472      }
473  
474      /* Print accumulated time */
475      printf("Total execution time: %.3f seconds\n", (double)(gettime_i64() - start_time) / 1000000);
476      if (tf->fn_teardown && tf->fn_teardown() != 0) return EXIT_FAILURE;
477  
478      return status;
479  }