pkg_repo_create.c
1 /*- 2 * Copyright (c) 2011-2025 Baptiste Daroussin <bapt@FreeBSD.org> 3 * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org> 4 * Copyright (c) 2011-2012 Marin Atanasov Nikolov <dnaeon@gmail.com> 5 * Copyright (c) 2012-2013 Matthew Seaman <matthew@FreeBSD.org> 6 * Copyright (c) 2014 Vsevolod Stakhov <vsevolod@FreeBSD.org> 7 * Copyright (c) 2023-2024 Serenity Cyber Security, LLC 8 * Author: Gleb Popov <arrowd@FreeBSD.org> 9 * 10 * SPDX-License-Identifier: BSD-2-Clause 11 */ 12 13 #include "pkg_config.h" 14 15 #include <sys/types.h> 16 #include <sys/stat.h> 17 #include <sys/uio.h> 18 #include <sys/file.h> 19 #include <sys/time.h> 20 #include <sys/wait.h> 21 22 #include <archive_entry.h> 23 #include <assert.h> 24 #include <fts.h> 25 #include <libgen.h> 26 #include <sqlite3.h> 27 #include <string.h> 28 #include <stdio.h> 29 #include <stdbool.h> 30 #include <unistd.h> 31 #include <errno.h> 32 #include <math.h> 33 #include <pthread.h> 34 #include <fcntl.h> 35 #include <dirent.h> 36 37 #include "pkg.h" 38 #include "private/event.h" 39 #include "private/utils.h" 40 #include "private/pkg.h" 41 #include "private/pkgdb.h" 42 #include "private/pkgsign.h" 43 44 enum { 45 MSG_PKG_DONE=0, 46 MSG_PKG_READY, 47 }; 48 49 static int pkg_repo_pack_db(const char *name, const char *archive, char *path, 50 struct pkgsign_ctx *ctx, struct pkg_repo_create *prc); 51 52 static int 53 hash_file(struct pkg_repo_meta *meta, struct pkg *pkg, char *path) 54 { 55 char tmp_repo[MAXPATHLEN] = { 0 }; 56 char tmp_name[MAXPATHLEN] = { 0 }; 57 char repo_name[MAXPATHLEN] = { 0 }; 58 char hash_name[MAXPATHLEN] = { 0 }; 59 char link_name[MAXPATHLEN] = { 0 }; 60 char *rel_repo = NULL; 61 char *rel_dir = NULL; 62 char *rel_link = NULL; 63 char *ext = NULL; 64 65 /* Don't rename symlinks */ 66 if (is_link(path)) 67 return (EPKG_OK); 68 69 ext = strrchr(path, '.'); 70 71 strlcpy(tmp_name, path, sizeof(tmp_name)); 72 rel_dir = get_dirname(tmp_name); 73 while (strstr(rel_dir, "/Hashed") != NULL) { 74 rel_dir = get_dirname(rel_dir); 75 } 76 strlcpy(tmp_name, rel_dir, sizeof(tmp_name)); 77 rel_dir = (char *)&tmp_name; 78 79 rel_repo = path; 80 if (strncmp(rel_repo, meta->repopath, strlen(meta->repopath)) == 0) { 81 rel_repo += strlen(meta->repopath); 82 while (rel_repo[0] == '/') 83 rel_repo++; 84 } 85 strlcpy(tmp_repo, rel_repo, sizeof(tmp_repo)); 86 rel_repo = get_dirname(tmp_repo); 87 while (strstr(rel_repo, "/Hashed") != NULL) { 88 rel_repo = get_dirname(rel_repo); 89 } 90 strlcpy(tmp_repo, rel_repo, sizeof(tmp_repo)); 91 rel_repo = (char *)&tmp_repo; 92 93 pkg_snprintf(repo_name, sizeof(repo_name), "%S/%S/%n-%v%S%z%S", 94 rel_repo, PKG_HASH_DIR, pkg, pkg, PKG_HASH_SEPSTR, pkg, ext); 95 pkg_snprintf(link_name, sizeof(repo_name), "%S/%n-%v%S", 96 rel_dir, pkg, pkg, ext); 97 pkg_snprintf(hash_name, sizeof(hash_name), "%S/%S/%n-%v%S%z%S", 98 rel_dir, PKG_HASH_DIR, pkg, pkg, PKG_HASH_SEPSTR, pkg, ext); 99 rel_link = (char *)&hash_name; 100 rel_link += strlen(rel_dir); 101 while (rel_link[0] == '/') 102 rel_link++; 103 104 snprintf(tmp_name, sizeof(tmp_name), "%s/%s", rel_dir, PKG_HASH_DIR); 105 rel_dir = (char *)&tmp_name; 106 if (!is_dir(rel_dir)) { 107 pkg_debug(1, "Making directory: %s", rel_dir); 108 (void)pkg_mkdirs(rel_dir); 109 } 110 111 if (!STREQ(path, hash_name)) { 112 pkg_debug(1, "Rename the pkg from: %s to: %s", path, hash_name); 113 if (rename(path, hash_name) == -1) { 114 pkg_emit_errno("rename", hash_name); 115 return (EPKG_FATAL); 116 } 117 } 118 if (meta->hash_symlink) { 119 pkg_debug(1, "Symlinking pkg file from: %s to: %s", rel_link, 120 link_name); 121 (void)unlink(link_name); 122 if (symlink(rel_link, link_name) == -1) { 123 pkg_emit_errno("symlink", link_name); 124 return (EPKG_FATAL); 125 } 126 } 127 free(pkg->repopath); 128 pkg->repopath = xstrdup(repo_name); 129 130 return (EPKG_OK); 131 } 132 133 struct pkg_fts_item { 134 char *fts_accpath; 135 char *pkg_path; 136 char *fts_name; 137 off_t fts_size; 138 int fts_info; 139 }; 140 typedef vec_t(struct pkg_fts_item *) fts_item_t; 141 142 static struct pkg_fts_item* 143 pkg_create_repo_fts_new(FTSENT *fts, const char *root_path) 144 { 145 struct pkg_fts_item *item; 146 char *pkg_path; 147 148 item = xmalloc(sizeof(*item)); 149 item->fts_accpath = xstrdup(fts->fts_accpath); 150 item->fts_name = xstrdup(fts->fts_name); 151 item->fts_size = fts->fts_statp->st_size; 152 item->fts_info = fts->fts_info; 153 154 pkg_path = fts->fts_path; 155 pkg_path += strlen(root_path); 156 while (pkg_path[0] == '/') 157 pkg_path++; 158 159 item->pkg_path = xstrdup(pkg_path); 160 161 return (item); 162 } 163 164 static void 165 pkg_create_repo_fts_free(struct pkg_fts_item *item) 166 { 167 free(item->fts_accpath); 168 free(item->pkg_path); 169 free(item->fts_name); 170 free(item); 171 } 172 173 static int 174 pkg_create_repo_read_fts(fts_item_t *items, FTS *fts, 175 const char *repopath, size_t *plen, struct pkg_repo_meta *meta) 176 { 177 FTSENT *fts_ent; 178 struct pkg_fts_item *fts_cur; 179 char *ext; 180 int linklen = 0; 181 char tmp_name[MAXPATHLEN] = { 0 }; 182 183 errno = 0; 184 185 while ((fts_ent = fts_read(fts)) != NULL) { 186 /* 187 * Skip directories starting with '.' to avoid Poudriere 188 * symlinks. 189 */ 190 if ((fts_ent->fts_info == FTS_D || 191 fts_ent->fts_info == FTS_DP) && 192 fts_ent->fts_namelen > 2 && 193 fts_ent->fts_name[0] == '.') { 194 fts_set(fts, fts_ent, FTS_SKIP); 195 continue; 196 } 197 /* 198 * Ignore 'Latest' directory as it is just symlinks back to 199 * already-processed packages. 200 */ 201 if ((fts_ent->fts_info == FTS_D || 202 fts_ent->fts_info == FTS_DP || 203 fts_ent->fts_info == FTS_SL) && 204 STREQ(fts_ent->fts_name, "Latest")) { 205 fts_set(fts, fts_ent, FTS_SKIP); 206 continue; 207 } 208 /* Follow symlinks. */ 209 if (fts_ent->fts_info == FTS_SL) { 210 /* Skip symlinks to hashed packages */ 211 if (meta->hash) { 212 linklen = readlink(fts_ent->fts_path, 213 (char *)&tmp_name, MAXPATHLEN); 214 if (linklen < 0) 215 continue; 216 tmp_name[linklen] = '\0'; 217 if (strstr(tmp_name, PKG_HASH_DIR) != NULL) 218 continue; 219 } 220 fts_set(fts, fts_ent, FTS_FOLLOW); 221 /* Restart. Next entry will be the resolved file. */ 222 continue; 223 } 224 /* Skip everything that is not a file */ 225 if (fts_ent->fts_info != FTS_F) 226 continue; 227 228 ext = strrchr(fts_ent->fts_name, '.'); 229 230 if (ext == NULL) 231 continue; 232 233 if (!packing_is_valid_format(ext + 1)) 234 continue; 235 236 /* skip all files which are not .pkg */ 237 if (!ctx.repo_accept_legacy_pkg && !STREQ(ext + 1, "pkg")) 238 continue; 239 240 241 *ext = '\0'; 242 243 if (pkg_repo_meta_is_old_file(fts_ent->fts_name, meta)) { 244 unlink(fts_ent->fts_path); 245 continue; 246 } 247 if (STREQ(fts_ent->fts_name, "meta") || 248 pkg_repo_meta_is_special_file(fts_ent->fts_name, meta)) { 249 *ext = '.'; 250 continue; 251 } 252 253 *ext = '.'; 254 fts_cur = pkg_create_repo_fts_new(fts_ent, repopath); 255 if (fts_cur == NULL) 256 return (EPKG_FATAL); 257 258 vec_push(items, fts_cur); 259 (*plen) ++; 260 } 261 262 if (errno != 0) { 263 pkg_emit_errno("fts_read", "pkg_create_repo_read_fts"); 264 return (EPKG_FATAL); 265 } 266 267 return (EPKG_OK); 268 } 269 270 struct thr_env { 271 int ntask; 272 FILE *ffile; 273 FILE *mfile; 274 FILE *dfile; 275 struct ucl_emitter_context *ctx; 276 struct pkg_repo_meta *meta; 277 fts_item_t fts_items; 278 pthread_mutex_t nlock; 279 pthread_mutex_t llock; 280 pthread_mutex_t flock; 281 pthread_cond_t cond; 282 }; 283 284 static void * 285 pkg_create_repo_thread(void *arg) 286 { 287 struct thr_env *te = (struct thr_env *)arg; 288 int flags, ret = EPKG_OK; 289 struct pkg *pkg = NULL; 290 char *path; 291 const char *repopath; 292 struct pkg_fts_item *items = NULL; 293 294 pkg_debug(1, "start worker to parse packages"); 295 296 if (te->ffile != NULL) 297 flags = PKG_OPEN_MANIFEST_ONLY; 298 else 299 flags = PKG_OPEN_MANIFEST_ONLY | PKG_OPEN_MANIFEST_COMPACT; 300 301 for (;;) { 302 if (items != NULL) 303 pkg_create_repo_fts_free(items); 304 pthread_mutex_lock(&te->llock); 305 if (te->fts_items.len == 0) { 306 pthread_mutex_unlock(&te->llock); 307 goto cleanup; 308 } 309 items = vec_pop(&te->fts_items); 310 pthread_mutex_unlock(&te->llock); 311 path = items->fts_accpath; 312 repopath = items->pkg_path; 313 if (pkg_open(&pkg, path, flags) == EPKG_OK) { 314 struct stat st; 315 316 pkg->sum = pkg_checksum_file(path, PKG_HASH_TYPE_SHA256_HEX); 317 stat(path, &st); 318 pkg->pkgsize = st.st_size; 319 if (te->meta->hash) { 320 ret = hash_file(te->meta, pkg, path); 321 if (ret != EPKG_OK) 322 goto cleanup; 323 } else { 324 pkg->repopath = xstrdup(repopath); 325 } 326 327 /* 328 * TODO: use pkg_checksum for new manifests 329 */ 330 pthread_mutex_lock(&te->flock); 331 ucl_object_t *o = pkg_emit_object(pkg, 0); 332 ucl_object_emit_streamline_add_object(te->ctx, o); 333 ucl_object_emit_file(o, UCL_EMIT_JSON_COMPACT, te->mfile); 334 fprintf(te->mfile, "\n"); 335 ucl_object_unref(o); 336 337 if (te->ffile != NULL) { 338 pkg_emit_filelist(pkg, te->ffile); 339 } 340 341 pthread_mutex_unlock(&te->flock); 342 343 pkg_free(pkg); 344 } 345 pthread_mutex_lock(&te->nlock); 346 te->ntask++; 347 pthread_cond_signal(&te->cond); 348 pthread_mutex_unlock(&te->nlock); 349 } 350 351 cleanup: 352 pkg_debug(1, "worker done"); 353 return (NULL); 354 } 355 356 #if defined (__linux__) || defined(_DARWIN_C_SOURCE) || defined (__APPLE__) 357 typedef const FTSENT *FTSENTP; 358 #else 359 typedef const FTSENT *const FTSENTP; 360 #endif 361 362 static int 363 fts_compare(FTSENTP *a, FTSENTP *b) 364 { 365 /* Sort files before directories, then alpha order */ 366 if ((*a)->fts_info != FTS_D && (*b)->fts_info == FTS_D) 367 return -1; 368 if ((*a)->fts_info == FTS_D && (*b)->fts_info != FTS_D) 369 return 1; 370 return (strcmp((*a)->fts_name, (*b)->fts_name)); 371 } 372 373 struct pkg_repo_create * 374 pkg_repo_create_new(void) 375 { 376 struct pkg_repo_create *prc; 377 378 prc = xcalloc(1, sizeof(*prc)); 379 prc->ofd = -1; 380 381 return (prc); 382 } 383 384 void 385 pkg_repo_create_free(struct pkg_repo_create *prc) 386 { 387 if (prc == NULL) 388 return; 389 pkg_repo_meta_free(prc->meta); 390 if (prc->ofd != -1) 391 close(prc->ofd); 392 ucl_object_unref(prc->groups); 393 free(prc); 394 } 395 396 static ucl_object_t* 397 ucl_load(int dfd, const char *name, ucl_object_t *schema) 398 { 399 struct ucl_parser *p; 400 ucl_object_t *obj = NULL; 401 int fd; 402 struct ucl_schema_error err; 403 404 fd = openat(dfd, name, O_RDONLY); 405 if (fd == -1) { 406 pkg_emit_error("Unable to open UCL file: %s", name); 407 return (NULL); 408 } 409 410 p = ucl_parser_new(0); 411 if (!ucl_parser_add_fd(p, fd)) { 412 pkg_emit_error("Error parsing UCL file '%s': %s'", 413 name, ucl_parser_get_error(p)); 414 ucl_parser_free(p); 415 close(fd); 416 return (NULL); 417 } 418 close(fd); 419 420 obj = ucl_parser_get_object(p); 421 ucl_parser_free(p); 422 if (obj == NULL) 423 return (NULL); 424 425 if (!ucl_object_validate(schema, obj, &err)) { 426 pkg_emit_error("UCL definition %s cannot be validated: %s", 427 name, err.msg); 428 ucl_object_unref(obj); 429 return (NULL); 430 } 431 432 return (obj); 433 } 434 435 static const char group_schema_str[] = "" 436 "{" 437 " type = object;" 438 " properties: {" 439 " name: { type = string };" 440 " requires: { " 441 " type = array;" 442 " item = { type = string };" 443 " };" 444 " depends: { " 445 " type = array;" 446 " item = { type = string };" 447 " };" 448 " comment: { type = string };" 449 " };" 450 " required = [ name, comment ];" 451 "};"; 452 453 static const char expired_schema_str[] = "" 454 "{" 455 " type = object;" 456 " properties: {" 457 " name: { type = string };" 458 " reason: { type = string };" 459 " replaced_by: { type = string };" 460 " };" 461 " required = [ name ];" 462 "};"; 463 464 static ucl_object_t * 465 open_schema(const char* schema_str, size_t schema_str_len) 466 { 467 struct ucl_parser *parser; 468 ucl_object_t *schema; 469 parser = ucl_parser_new(UCL_PARSER_NO_FILEVARS); 470 if (!ucl_parser_add_chunk(parser, schema_str, 471 schema_str_len - 1)) { 472 pkg_emit_error("Cannot parse schema string: %s", 473 ucl_parser_get_error(parser)); 474 ucl_parser_free(parser); 475 return (NULL); 476 } 477 schema = ucl_parser_get_object(parser); 478 ucl_parser_free(parser); 479 return (schema); 480 } 481 482 static void 483 read_ucl_dir(struct pkg_repo_create *prc, const char *path, ucl_object_t *schema, void (*callback)(struct pkg_repo_create *prc, ucl_object_t* parsed_obj)) 484 { 485 int dfd = open(path, O_DIRECTORY); 486 DIR *d; 487 struct dirent *e; 488 struct stat st; 489 490 if (dfd == -1) { 491 pkg_emit_error("Unable to open directory '%s'", path); 492 return; 493 } 494 495 d = fdopendir(dfd); 496 if (d == NULL) { 497 pkg_emit_error("Unable to open directory '%s'", path); 498 close(dfd); 499 return; 500 } 501 502 while ((e = readdir(d)) != NULL) { 503 const char *ext; 504 ucl_object_t* parsed_obj; 505 /* ignore all hidden files */ 506 if (e->d_name[0] == '.') 507 continue; 508 /* only consider files ending with .ucl */ 509 ext = strrchr(e->d_name, '.'); 510 if (ext == NULL) 511 continue; 512 if (strcmp(ext, ".ucl") != 0) 513 continue; 514 /* only regular files are considered */ 515 if (fstatat(dfd, e->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) { 516 pkg_emit_errno("fstatat", e->d_name); 517 goto cleanup; 518 } 519 if (!S_ISREG(st.st_mode)) 520 continue; 521 parsed_obj = ucl_load(dfd, e->d_name, schema); 522 if (parsed_obj) 523 callback (prc, parsed_obj); 524 } 525 cleanup: 526 closedir(d); 527 } 528 529 static void 530 append_groups(struct pkg_repo_create *prc, ucl_object_t* groups_obj) 531 { 532 if (prc->groups == NULL) 533 prc->groups = ucl_object_typed_new(UCL_ARRAY); 534 ucl_array_append(prc->groups, groups_obj); 535 } 536 537 void 538 pkg_repo_create_set_groups(struct pkg_repo_create *prc, const char *path) 539 { 540 ucl_object_t *schema; 541 schema = open_schema(group_schema_str, sizeof(group_schema_str)); 542 543 read_ucl_dir(prc, path, schema, append_groups); 544 545 ucl_object_unref(schema); 546 } 547 548 static void 549 append_expired_packages(struct pkg_repo_create *prc, ucl_object_t* expired_packages_obj) 550 { 551 if (prc->expired_packages == NULL) 552 prc->expired_packages = ucl_object_typed_new(UCL_ARRAY); 553 ucl_array_append(prc->expired_packages, expired_packages_obj); 554 } 555 556 void 557 pkg_repo_create_set_expired_packages(struct pkg_repo_create *prc, const char *path) 558 { 559 ucl_object_t *schema; 560 schema = open_schema(expired_schema_str, sizeof(expired_schema_str)); 561 562 read_ucl_dir(prc, path, schema, append_expired_packages); 563 564 ucl_object_unref(schema); 565 } 566 567 void 568 pkg_repo_create_set_create_filelist(struct pkg_repo_create *prc, bool val) 569 { 570 prc->filelist = val; 571 } 572 573 void 574 pkg_repo_create_set_hash(struct pkg_repo_create *prc, bool val) 575 { 576 prc->hash = val; 577 } 578 579 void 580 pkg_repo_create_set_hash_symlink(struct pkg_repo_create *prc, bool val) 581 { 582 prc->hash_symlink = val; 583 } 584 585 void 586 pkg_repo_create_set_output_dir(struct pkg_repo_create *prc, const char *out) 587 { 588 prc->outdir = out; 589 } 590 591 void 592 pkg_repo_create_set_metafile(struct pkg_repo_create *prc, const char *metafile) 593 { 594 prc->metafile = metafile; 595 } 596 597 void 598 pkg_repo_create_set_sign(struct pkg_repo_create *prc, char **argv, int argc, pkg_password_cb *cb) 599 { 600 prc->sign.argc = argc; 601 prc->sign.argv = argv; 602 prc->sign.cb = cb; 603 } 604 605 static int 606 pkg_repo_create_pack_and_sign(struct pkg_repo_create *prc) 607 { 608 char repo_path[MAXPATHLEN]; 609 char repo_archive[MAXPATHLEN]; 610 char *key_file; 611 const char *key_type; 612 struct pkgsign_ctx *sctx = NULL; 613 struct stat st; 614 int ret = EPKG_OK, nfile = 0; 615 const int files_to_pack = 4; 616 617 if (prc->sign.argc == 1) { 618 char *cpos; 619 620 key_type = key_file = prc->sign.argv[0]; 621 if ((cpos = strchr(key_type, ':')) != NULL) { 622 key_file = cpos + 1; 623 *(key_file - 1) = '\0'; 624 } else { 625 key_type = "rsa"; 626 } 627 628 pkg_debug(1, "Loading %s key from '%s' for signing", key_type, key_file); 629 ret = pkgsign_new_sign(key_type, &sctx); 630 if (ret != EPKG_OK) { 631 pkg_emit_error("'%s' signer not found", key_type); 632 return (EPKG_FATAL); 633 } 634 635 pkgsign_set(sctx, prc->sign.cb, key_file); 636 ret = EPKG_OK; 637 } 638 639 if (prc->sign.argc > 1 && !STREQ(prc->sign.argv[0], "signing_command:")) 640 return (EPKG_FATAL); 641 642 if (prc->sign.argc > 1) { 643 prc->sign.argc--; 644 prc->sign.argv++; 645 } 646 647 pkg_emit_progress_start("Packing files for repository"); 648 pkg_emit_progress_tick(nfile++, files_to_pack); 649 650 snprintf(repo_path, sizeof(repo_path), "%s/%s", prc->outdir, 651 prc->meta->manifests); 652 snprintf(repo_archive, sizeof(repo_archive), "%s/%s", prc->outdir, 653 prc->meta->manifests_archive); 654 if (pkg_repo_pack_db(prc->meta->manifests, repo_archive, repo_path, sctx, prc) != EPKG_OK) { 655 ret = EPKG_FATAL; 656 goto cleanup; 657 } 658 659 pkg_emit_progress_tick(nfile++, files_to_pack); 660 661 if (prc->filelist) { 662 snprintf(repo_path, sizeof(repo_path), "%s/%s", prc->outdir, 663 prc->meta->filesite); 664 snprintf(repo_archive, sizeof(repo_archive), "%s/%s", 665 prc->outdir, prc->meta->filesite_archive); 666 if (pkg_repo_pack_db(prc->meta->filesite, repo_archive, repo_path, sctx, prc) != EPKG_OK) { 667 ret = EPKG_FATAL; 668 goto cleanup; 669 } 670 } 671 672 pkg_emit_progress_tick(nfile++, files_to_pack); 673 snprintf(repo_path, sizeof(repo_path), "%s/%s", prc->outdir, prc->meta->data); 674 snprintf(repo_archive, sizeof(repo_archive), "%s/%s", prc->outdir, 675 prc->meta->data_archive); 676 if (pkg_repo_pack_db(prc->meta->data, repo_archive, repo_path, sctx, prc) != EPKG_OK) { 677 ret = EPKG_FATAL; 678 goto cleanup; 679 } 680 681 pkg_emit_progress_tick(nfile++, files_to_pack); 682 683 if (fstatat(prc->ofd, "meta.conf", &st, 0) == 0) { 684 struct timespec ts[2] = { 685 { 686 .tv_sec = st.st_mtime, 687 .tv_nsec = 0, 688 }, 689 { 690 .tv_sec = st.st_mtime, 691 .tv_nsec = 0, 692 }, 693 }; 694 snprintf(repo_archive, sizeof(repo_archive), "%s.pkg", 695 prc->meta->manifests_archive); 696 utimensat(prc->ofd, repo_archive, ts, 0); 697 if (prc->filelist) { 698 snprintf(repo_archive, sizeof(repo_archive), 699 "%s.pkg", prc->meta->filesite_archive); 700 utimensat(prc->ofd, repo_archive, ts, 0); 701 } 702 snprintf(repo_archive, sizeof(repo_archive), "%s.pkg", 703 prc->meta->data_archive); 704 utimensat(prc->ofd, repo_archive, ts, 0); 705 } 706 707 cleanup: 708 pkg_emit_progress_tick(files_to_pack, files_to_pack); 709 710 pkgsign_free(sctx); 711 712 return (ret); 713 } 714 715 int 716 pkg_repo_create(struct pkg_repo_create *prc, char *path) 717 { 718 FTS *fts = NULL; 719 int num_workers; 720 pthread_t *threads; 721 struct thr_env te = { 0 }; 722 size_t len; 723 int fd; 724 int dfd, ffd, mfd; 725 int retcode = EPKG_FATAL; 726 ucl_object_t *meta_dump; 727 char *repopath[2]; 728 729 if (prc->outdir == NULL) 730 prc->outdir = path; 731 732 te.dfile = te.ffile = te.mfile = NULL; 733 734 if (!is_dir(path)) { 735 pkg_emit_error("%s is not a directory", path); 736 return (EPKG_FATAL); 737 } 738 739 errno = 0; 740 if (!is_dir(prc->outdir)) { 741 /* Try to create dir */ 742 if (errno == ENOENT) { 743 if (mkdir(prc->outdir, 00755) == -1) { 744 pkg_fatal_errno("cannot create output directory %s", 745 prc->outdir); 746 } 747 } 748 else { 749 pkg_emit_error("%s is not a directory", prc->outdir); 750 return (EPKG_FATAL); 751 } 752 } 753 if ((prc->ofd = open(prc->outdir, O_DIRECTORY)) == -1) { 754 pkg_emit_error("Cannot open %s", prc->outdir); 755 return (EPKG_FATAL); 756 } 757 758 if (prc->metafile != NULL) { 759 fd = open(prc->metafile, O_RDONLY); 760 if (fd == -1) { 761 pkg_emit_error("meta loading error while trying %s", prc->metafile); 762 return (EPKG_FATAL); 763 } 764 if (pkg_repo_meta_load(fd, &prc->meta) != EPKG_OK) { 765 pkg_emit_error("meta loading error while trying %s", prc->metafile); 766 close(fd); 767 return (EPKG_FATAL); 768 } 769 close(fd); 770 } else { 771 prc->meta = pkg_repo_meta_default(); 772 } 773 prc->meta->repopath = path; 774 prc->meta->hash = prc->hash; 775 prc->meta->hash_symlink = prc->hash_symlink; 776 777 te.meta = prc->meta; 778 779 /* initialize mutexes & conditions */ 780 pthread_mutex_init(&te.nlock, 0); 781 pthread_mutex_init(&te.llock, 0); 782 pthread_mutex_init(&te.flock, 0); 783 pthread_cond_init(&te.cond, 0); 784 785 repopath[0] = path; 786 repopath[1] = NULL; 787 788 num_workers = pkg_object_int(pkg_config_get("WORKERS_COUNT")); 789 if (num_workers <= 0) { 790 num_workers = (int)sysconf(_SC_NPROCESSORS_ONLN); 791 if (num_workers == -1) 792 num_workers = 6; 793 } 794 795 if ((fts = fts_open(repopath, FTS_PHYSICAL|FTS_NOCHDIR, fts_compare)) == NULL) { 796 pkg_emit_errno("fts_open", path); 797 goto cleanup; 798 } 799 800 if ((mfd = openat(prc->ofd, prc->meta->manifests, 801 O_CREAT|O_TRUNC|O_WRONLY, 00644)) == -1) { 802 goto cleanup; 803 } 804 if ((te.mfile = fdopen(mfd,"w")) == NULL) { 805 goto cleanup; 806 } 807 808 if ((dfd = openat(prc->ofd, prc->meta->data, 809 O_CREAT|O_TRUNC|O_WRONLY, 00644)) == -1) { 810 goto cleanup; 811 } 812 if ((te.dfile = fdopen(dfd,"w")) == NULL) { 813 goto cleanup; 814 } 815 816 if (prc->filelist) { 817 if ((ffd = openat(prc->ofd, prc->meta->filesite, 818 O_CREAT|O_TRUNC|O_WRONLY, 00644)) == -1) { 819 goto cleanup; 820 } 821 if ((te.ffile = fdopen(ffd,"w")) == NULL) { 822 goto cleanup; 823 } 824 } 825 826 len = 0; 827 828 pkg_create_repo_read_fts(&te.fts_items, fts, path, &len, prc->meta); 829 830 if (len == 0) { 831 /* Nothing to do */ 832 pkg_emit_error("No package files have been found"); 833 goto cleanup; 834 } 835 836 /* Split items over all workers */ 837 num_workers = MIN(num_workers, len); 838 839 /* Launch workers */ 840 pkg_emit_progress_start("Creating repository in %s", prc->outdir); 841 842 threads = xcalloc(num_workers, sizeof(pthread_t)); 843 844 struct ucl_emitter_functions *f; 845 ucl_object_t *obj = ucl_object_typed_new(UCL_OBJECT); 846 /* 847 * Work around a bug in the streamline exporter which creates an invalid 848 * json if there is nothing in the object, prior to the streamline to 849 * start. So always add at least an empty groups array 850 */ 851 ucl_object_insert_key(obj, 852 prc->groups == NULL ? ucl_object_typed_new(UCL_ARRAY) : prc->groups, 853 "groups", 0, false); 854 ucl_object_insert_key(obj, 855 prc->expired_packages == NULL ? ucl_object_typed_new(UCL_ARRAY) : prc->expired_packages, 856 "expired_packages", 0, false); 857 f = ucl_object_emit_file_funcs(te.dfile); 858 te.ctx = ucl_object_emit_streamline_new(obj, UCL_EMIT_JSON_COMPACT, f); 859 ucl_object_t *ar = ucl_object_typed_new(UCL_ARRAY); 860 ar->key = "packages"; 861 ar->keylen = sizeof("packages") -1; 862 863 ucl_object_emit_streamline_start_container(te.ctx, ar); 864 865 for (int i = 0; i < num_workers; i++) { 866 /* Create new worker */ 867 pthread_create(&threads[i], NULL, &pkg_create_repo_thread, &te); 868 } 869 870 pthread_mutex_lock(&te.nlock); 871 while (te.ntask < len) { 872 pthread_cond_wait(&te.cond, &te.nlock); 873 pkg_emit_progress_tick(te.ntask, len); 874 } 875 pthread_mutex_unlock(&te.nlock); 876 877 for (int i = 0; i < num_workers; i++) 878 pthread_join(threads[i], NULL); 879 free(threads); 880 ucl_object_emit_streamline_end_container(te.ctx); 881 pkg_emit_progress_tick(len, len); 882 ucl_object_emit_streamline_finish(te.ctx); 883 ucl_object_emit_funcs_free(f); 884 ucl_object_unref(obj); 885 ucl_object_unref(ar); 886 887 /* Write metafile */ 888 889 fd = openat(prc->ofd, "meta", O_CREAT|O_TRUNC|O_CLOEXEC|O_WRONLY, 890 0644); 891 if (fd != -1) { 892 meta_dump = pkg_repo_meta_to_ucl(prc->meta); 893 ucl_object_emit_fd(meta_dump, UCL_EMIT_CONFIG, fd); 894 close(fd); 895 fd = openat(prc->ofd, "meta.conf", 896 O_CREAT|O_TRUNC|O_CLOEXEC|O_WRONLY, 0644); 897 if (fd != -1) { 898 ucl_object_emit_fd(meta_dump, UCL_EMIT_CONFIG, fd); 899 close(fd);; 900 } else { 901 pkg_emit_notice("cannot create metafile at 'meta.conf'"); 902 } 903 ucl_object_unref(meta_dump); 904 } 905 else { 906 pkg_emit_notice("cannot create metafile at 'meta'"); 907 } 908 retcode = EPKG_OK; 909 cleanup: 910 if (te.mfile != NULL) 911 fclose(te.mfile); 912 if (te.ffile != NULL) 913 fclose(te.ffile); 914 if (te.dfile != NULL) 915 fclose(te.dfile); 916 if (fts != NULL) 917 fts_close(fts); 918 919 vec_free_and_free(&te.fts_items, pkg_create_repo_fts_free); 920 921 if (retcode != EPKG_OK) 922 return (retcode); 923 924 return (pkg_repo_create_pack_and_sign(prc)); 925 } 926 927 static int 928 pkg_repo_sign(const char *path, char **argv, int argc, char **sig, size_t *siglen, 929 char **sigtype, char **cert, size_t *certlen) 930 { 931 FILE *fps[2]; 932 char *sha256; 933 xstring *cmd = NULL; 934 xstring *buf = NULL; 935 xstring *sigstr = NULL; 936 xstring *certstr = NULL; 937 xstring *typestr = NULL; 938 char *line = NULL; 939 size_t linecap = 0; 940 ssize_t linelen; 941 pid_t spid; 942 int i, pstatus, ret = EPKG_OK; 943 bool end_seen = false; 944 945 sha256 = pkg_checksum_file(path, PKG_HASH_TYPE_SHA256_HEX); 946 if (!sha256) 947 return (EPKG_FATAL); 948 949 cmd = xstring_new(); 950 951 for (i = 0; i < argc; i++) { 952 if (strspn(argv[i], " \t\n") > 0) 953 fprintf(cmd->fp, " \"%s\" ", argv[i]); 954 else 955 fprintf(cmd->fp, " %s ", argv[i]); 956 } 957 958 fflush(cmd->fp); 959 if ((spid = process_spawn_pipe(fps, cmd->buf)) < 0) { 960 ret = EPKG_FATAL; 961 goto done; 962 } 963 964 fprintf(fps[1], "%s\n", sha256); 965 fflush(fps[1]); 966 967 sigstr = xstring_new(); 968 certstr = xstring_new(); 969 typestr = xstring_new(); 970 971 while ((linelen = getline(&line, &linecap, fps[0])) > 0 ) { 972 if (STREQ(line, "SIGNATURE\n")) { 973 buf = sigstr; 974 continue; 975 } else if (STREQ(line, "CERT\n")) { 976 buf = certstr; 977 continue; 978 } else if (STREQ(line, "TYPE\n")) { 979 buf = typestr; 980 continue; 981 } else if (STREQ(line, "END\n")) { 982 end_seen = true; 983 break; 984 } 985 if (buf != NULL) { 986 fwrite(line, linelen, 1, buf->fp); 987 } 988 } 989 free(line); 990 991 *sigtype = xstring_get(typestr); 992 *cert = xstring_get_binary(certstr, certlen); 993 *sig = xstring_get_binary(sigstr, siglen); 994 995 /* 996 * cert could be DER-encoded rather than PEM, so strip off any trailing 997 * END marker if we ran over it. 998 */ 999 if (!end_seen && *certlen >= 4 && 1000 STREQ(&(*cert)[*certlen - 4], "END\n")) 1001 *certlen -= 4; 1002 1003 /* remove the latest \n */ 1004 *siglen -= 1; 1005 1006 waitpid(spid, &pstatus, WNOHANG); 1007 fclose(fps[0]); 1008 fclose(fps[1]); 1009 1010 done: 1011 free(sha256); 1012 xstring_free(cmd); 1013 1014 return (ret); 1015 } 1016 1017 static int 1018 pack_sign(struct packing *pack, struct pkgsign_ctx *sctx, const char *path, 1019 const char *name) 1020 { 1021 unsigned char *sigret = NULL; 1022 const char *sigtype; 1023 size_t siglen = 0; 1024 struct iovec iov[2]; 1025 char buf[32]; 1026 int offset, size; 1027 1028 if (sctx == NULL) 1029 return (EPKG_FATAL); 1030 1031 if (pkgsign_sign(sctx, path, &sigret, &siglen) != EPKG_OK) { 1032 free(sigret); 1033 return (EPKG_FATAL); 1034 } 1035 1036 offset = 0; 1037 sigtype = pkgsign_impl_name(sctx); 1038 if (!STREQ(sigtype, "rsa")) { 1039 size = snprintf(buf, sizeof(buf), "%s%s$", PKGSIGN_HEAD, sigtype); 1040 if (size >= (int)sizeof(buf)) { 1041 free(sigret); 1042 return (EPKG_FATAL); 1043 } 1044 1045 iov[offset].iov_base = buf; 1046 iov[offset++].iov_len = size; 1047 } 1048 1049 iov[offset].iov_base = sigret; 1050 iov[offset++].iov_len = siglen; 1051 1052 if (packing_append_iovec(pack, name, iov, offset) != EPKG_OK) { 1053 free(sigret); 1054 return (EPKG_FATAL); 1055 } 1056 free(sigret); 1057 1058 return (EPKG_OK); 1059 } 1060 1061 static int 1062 pack_command_sign(struct packing *pack, const char *path, char **argv, int argc, 1063 const char *name) 1064 { 1065 size_t pub_len = 0, signature_len = 0; 1066 char fname[MAXPATHLEN]; 1067 char *sig, *sigtype, *pub; 1068 char buf[32]; 1069 struct iovec iov[2]; 1070 int offset, size; 1071 1072 sig = NULL; 1073 pub = NULL; 1074 1075 if (pkg_repo_sign(path, argv, argc, &sig, &signature_len, &sigtype, &pub, 1076 &pub_len) != EPKG_OK) { 1077 free(sig); 1078 free(pub); 1079 return (EPKG_FATAL); 1080 } 1081 1082 offset = 0; 1083 snprintf(fname, sizeof(fname), "%s.sig", name); 1084 if (*sigtype != '\0' && !STREQ(sigtype, "rsa")) { 1085 int typelen; 1086 1087 typelen = strlen(sigtype); 1088 if (sigtype[typelen - 1] == '\n') 1089 sigtype[--typelen] = '\0'; 1090 size = snprintf(buf, sizeof(buf), "%s%s$", PKGSIGN_HEAD, sigtype); 1091 free(sigtype); 1092 if (size >= (int)sizeof(buf)) { 1093 free(sig); 1094 free(pub); 1095 return (EPKG_FATAL); 1096 } 1097 1098 iov[offset].iov_base = buf; 1099 iov[offset++].iov_len = size; 1100 } else { 1101 free(sigtype); 1102 } 1103 1104 iov[offset].iov_base = sig; 1105 iov[offset].iov_len = signature_len; 1106 1107 if (packing_append_iovec(pack, fname, iov, offset + 1) != EPKG_OK) { 1108 free(sig); 1109 free(pub); 1110 return (EPKG_FATAL); 1111 } 1112 free(sig); 1113 1114 snprintf(fname, sizeof(fname), "%s.pub", name); 1115 iov[offset].iov_base = pub; 1116 iov[offset].iov_len = pub_len; 1117 if (packing_append_iovec(pack, fname, iov, offset + 1) != EPKG_OK) { 1118 free(pub); 1119 return (EPKG_FATAL); 1120 } 1121 free(pub); 1122 1123 return (EPKG_OK); 1124 } 1125 1126 static int 1127 pkg_repo_pack_db(const char *name, const char *archive, char *path, 1128 struct pkgsign_ctx *sctx, struct pkg_repo_create *prc) 1129 { 1130 struct packing *pack; 1131 int ret = EPKG_OK; 1132 1133 if (packing_init(&pack, archive, prc->meta->packing_format, 0, 0, (time_t)-1, true) != EPKG_OK) 1134 return (EPKG_FATAL); 1135 1136 if (sctx != NULL) { 1137 ret = pack_sign(pack, sctx, path, "signature"); 1138 } else if (prc->sign.argc >= 1) { 1139 ret = pack_command_sign(pack, path, prc->sign.argv, prc->sign.argc, name); 1140 } 1141 packing_append_file_attr(pack, path, name, "root", "wheel", 0644, 0); 1142 1143 packing_finish(pack); 1144 unlink(path); 1145 1146 return (ret); 1147 }