pkg_create.c
1 /*- 2 * Copyright (c) 2011-2020 Baptiste Daroussin <bapt@FreeBSD.org> 3 * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org> 4 * Copyright (c) 2014-2015 Matthew Seaman <matthew@FreeBSD.org> 5 * Copyright (c) 2014 Vsevolod Stakhov <vsevolod@FreeBSD.org> 6 * Copyright (c) 2023 Serenity Cyber Security, LLC 7 * Author: Gleb Popov <arrowd@FreeBSD.org> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer 15 * in this position and unchanged. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/stat.h> 33 34 #include <errno.h> 35 #include <regex.h> 36 #include <fcntl.h> 37 38 #include <bsd_compat.h> 39 40 #include "pkg.h" 41 #include "private/event.h" 42 #include "private/pkg.h" 43 #include "private/pkg_abi.h" 44 #include "xmalloc.h" 45 46 #define TICK 100 47 48 static int load_metadata(struct pkg *pkg, const char *metadata, const char *plist, 49 const char *rootdir); 50 static void fixup_abi(struct pkg *pkg, const char *rootdir, bool testing); 51 static void counter_init(const char *what, int64_t max); 52 static void counter_count(void); 53 static void counter_end(void); 54 55 extern struct pkg_ctx ctx; 56 57 static int 58 pkg_create_from_dir(struct pkg *pkg, const char *root, 59 struct pkg_create *pc, struct packing *pkg_archive, 60 bool trust_filesystem) 61 { 62 char fpath[MAXPATHLEN]; 63 struct pkg_file *file = NULL; 64 struct pkg_dir *dir = NULL; 65 int ret; 66 struct stat st; 67 int64_t flatsize = 0; 68 int64_t nfiles; 69 const char *relocation; 70 char *manifest; 71 ucl_object_t *obj; 72 hardlinks_t hardlinks = vec_init(); 73 ssize_t linklen; 74 75 if (pkg_is_valid(pkg) != EPKG_OK) { 76 pkg_emit_error("the package is not valid"); 77 return (EPKG_FATAL); 78 } 79 80 relocation = pkg_kv_get(&pkg->annotations, "relocated"); 81 if (relocation == NULL) 82 relocation = ""; 83 if (ctx.pkg_rootdir != NULL) 84 relocation = ctx.pkg_rootdir; 85 86 /* 87 * Get / compute size / checksum if not provided in the manifest 88 */ 89 90 nfiles = pkghash_count(pkg->filehash); 91 counter_init("file sizes/checksums", nfiles); 92 93 while (pkg_files(pkg, &file) == EPKG_OK) { 94 95 snprintf(fpath, sizeof(fpath), "%s%s%s", root ? root : "", 96 relocation, file->path); 97 98 if (lstat(fpath, &st) == -1) { 99 pkg_emit_error("file '%s' is missing", fpath); 100 vec_free_and_free(&hardlinks, free); 101 return (EPKG_FATAL); 102 } 103 104 if (!(S_ISREG(st.st_mode) || S_ISLNK(st.st_mode))) { 105 pkg_emit_error("file '%s' is not a regular file or symlink", fpath); 106 vec_free_and_free(&hardlinks, free); 107 return (EPKG_FATAL); 108 } 109 110 if (file->size == 0) 111 file->size = (int64_t)st.st_size; 112 113 if (st.st_nlink == 1 || !check_for_hardlink(&hardlinks, &st)) { 114 flatsize += file->size; 115 } 116 117 if (file->perm == 0) { 118 file->perm = st.st_mode & ~S_IFMT; 119 } 120 121 if (trust_filesystem) { 122 free(file->sum); 123 file->sum = pkg_checksum_generate_file(fpath, 124 PKG_HASH_TYPE_SHA256_HEX); 125 if (file->sum == NULL) { 126 vec_free_and_free(&hardlinks, free); 127 return (EPKG_FATAL); 128 } 129 130 if (S_ISLNK(st.st_mode)) { 131 char link[MAXPATHLEN] = { 0 }; 132 linklen = readlink(fpath, link, sizeof(link) -1); 133 if (linklen == -1) { 134 vec_free_and_free(&hardlinks, free); 135 pkg_emit_errno("pkg_create_from_dir", "readlink failed"); 136 return (EPKG_FATAL); 137 } 138 free(file->symlink_target); 139 file->symlink_target = xstrdup(link); 140 } 141 142 if (pc->timestamp > (time_t)-1) { 143 file->time[0].tv_sec = pc->timestamp; 144 file->time[1].tv_sec = pc->timestamp; 145 } else { 146 file->time[0] = st.st_atim; 147 file->time[1] = st.st_mtim; 148 } 149 } 150 151 counter_count(); 152 } 153 vec_free_and_free(&hardlinks, free); 154 155 while (pkg_dirs(pkg, &dir) == EPKG_OK) { 156 snprintf(fpath, sizeof(fpath), "%s%s%s", root ? root : "", 157 relocation, dir->path); 158 159 if (lstat(fpath, &st) == -1) { 160 pkg_emit_error("dir '%s' is missing", fpath); 161 return (EPKG_FATAL); 162 } 163 164 if (!S_ISDIR(st.st_mode)) { 165 pkg_emit_error("dir '%s' is not a directory", fpath); 166 return (EPKG_FATAL); 167 } 168 169 if (dir->perm == 0) { 170 dir->perm = st.st_mode & ~S_IFMT; 171 } 172 } 173 174 counter_end(); 175 176 pkg->flatsize = flatsize; 177 178 if (pkg->type == PKG_OLD_FILE) { 179 pkg_emit_error("Cannot create an old format package"); 180 return (EPKG_FATAL); 181 } 182 183 obj = pkg_emit_object(pkg, PKG_MANIFEST_EMIT_COMPACT); 184 manifest = ucl_object_emit(obj, UCL_EMIT_JSON_COMPACT); 185 ucl_object_unref(obj); 186 packing_append_buffer(pkg_archive, manifest, "+COMPACT_MANIFEST", strlen(manifest)); 187 free(manifest); 188 obj = pkg_emit_object(pkg, 0); 189 if (pc->expand_manifest) { 190 manifest = ucl_object_emit(obj, UCL_EMIT_CONFIG); 191 } else { 192 manifest = ucl_object_emit(obj, UCL_EMIT_JSON_COMPACT); 193 } 194 ucl_object_unref(obj); 195 packing_append_buffer(pkg_archive, manifest, "+MANIFEST", strlen(manifest)); 196 free(manifest); 197 198 counter_init("packing files", nfiles); 199 200 while (pkg_files(pkg, &file) == EPKG_OK) { 201 char dpath[MAXPATHLEN]; 202 const char *dp = file->path; 203 204 if (pkg->oprefix != NULL) { 205 size_t l = strlen(pkg->prefix); 206 if (strncmp(file->path, pkg->prefix, l) == 0 && 207 (file->path[l] == '/' || l == 1)) { 208 snprintf(dpath, sizeof(dpath), "%s%s%s", 209 pkg->oprefix, l == 1 ? "/" : "", file->path + l); 210 dp = dpath; 211 } 212 } 213 214 snprintf(fpath, sizeof(fpath), "%s%s%s", root ? root : "", 215 relocation, file->path); 216 217 ret = packing_append_file_attr(pkg_archive, fpath, dp, 218 file->uname, file->gname, file->perm, file->fflags); 219 if (ctx.developer_mode && ret != EPKG_OK) 220 return (ret); 221 counter_count(); 222 } 223 224 counter_end(); 225 226 nfiles = pkghash_count(pkg->dirhash); 227 counter_init("packing directories", nfiles); 228 229 while (pkg_dirs(pkg, &dir) == EPKG_OK) { 230 snprintf(fpath, sizeof(fpath), "%s%s%s", root ? root : "", 231 relocation, dir->path); 232 233 ret = packing_append_file_attr(pkg_archive, fpath, dir->path, 234 dir->uname, dir->gname, dir->perm, dir->fflags); 235 if (ctx.developer_mode && ret != EPKG_OK) 236 return (ret); 237 counter_count(); 238 } 239 240 counter_end(); 241 242 return (EPKG_OK); 243 } 244 245 static struct packing * 246 pkg_create_archive(struct pkg *pkg, struct pkg_create *pc, unsigned required_flags) 247 { 248 char *pkg_path = NULL; 249 struct packing *pkg_archive = NULL; 250 251 /* 252 * Ensure that we have all the information we need 253 */ 254 if (pkg->type != PKG_OLD_FILE) 255 assert((pkg->flags & required_flags) == required_flags); 256 257 if (pkg_mkdirs(pc->outdir) != EPKG_OK) 258 return NULL; 259 260 if (pkg_asprintf(&pkg_path, "%S/%n-%v", pc->outdir, pkg, pkg) == -1) { 261 pkg_emit_errno("pkg_asprintf", ""); 262 return (NULL); 263 } 264 265 if (packing_init(&pkg_archive, pkg_path, pc->format, 266 pc->compression_level, pc->compression_threads, pc->timestamp, pc->overwrite) != EPKG_OK) { 267 pkg_archive = NULL; 268 } 269 270 free(pkg_path); 271 272 return pkg_archive; 273 } 274 275 static const char * const scripts[] = { 276 "+INSTALL", 277 "+PRE_INSTALL", 278 "+POST_INSTALL", 279 "+POST_INSTALL", 280 "+DEINSTALL", 281 "+PRE_DEINSTALL", 282 "+POST_DEINSTALL", 283 "pkg-install", 284 "pkg-pre-install", 285 "pkg-post-install", 286 "pkg-deinstall", 287 "pkg-pre-deinstall", 288 "pkg-post-deinstall", 289 NULL 290 }; 291 292 static const char * const lua_scripts[] = { 293 "pkg-pre-install.lua", 294 "pkg-post-install.lua", 295 "pkg-pre-deinstall.lua", 296 "pkg-post-deinstall.lua", 297 NULL 298 }; 299 300 struct pkg_create * 301 pkg_create_new(void) 302 { 303 struct pkg_create *pc; 304 305 pc = xcalloc(1, sizeof(*pc)); 306 pc->format = packing_format_from_string(ctx.compression_format); 307 pc->compression_level = ctx.compression_level; 308 pc->compression_threads = ctx.compression_threads; 309 pc->timestamp = (time_t) -1; 310 pc->overwrite = true; 311 pc->expand_manifest = false; 312 313 return (pc); 314 } 315 316 void 317 pkg_create_free(struct pkg_create *pc) 318 { 319 free(pc); 320 } 321 322 bool 323 pkg_create_set_format(struct pkg_create *pc, const char *format) 324 { 325 if (STREQ(format, "tzst")) 326 pc->format = TZS; 327 else if (STREQ(format, "txz")) 328 pc->format = TXZ; 329 else if (STREQ(format, "tbz")) 330 pc->format = TBZ; 331 else if (STREQ(format, "tgz")) 332 pc->format = TGZ; 333 else if (STREQ(format, "tar")) 334 pc->format = TAR; 335 else 336 return (false); 337 return (true); 338 } 339 340 void 341 pkg_create_set_compression_level(struct pkg_create *pc, int clevel) 342 { 343 pc->compression_level = clevel; 344 } 345 346 void 347 pkg_create_set_compression_threads(struct pkg_create *pc, int threads) 348 { 349 pc->compression_threads = threads; 350 } 351 352 void 353 pkg_create_set_expand_manifest(struct pkg_create *pc, bool expand) 354 { 355 pc->expand_manifest = expand; 356 } 357 358 void 359 pkg_create_set_rootdir(struct pkg_create *pc, const char *rootdir) 360 { 361 pc->rootdir = rootdir; 362 } 363 364 void 365 pkg_create_set_output_dir(struct pkg_create *pc, const char *outdir) 366 { 367 pc->outdir = outdir; 368 } 369 370 void 371 pkg_create_set_timestamp(struct pkg_create *pc, time_t timestamp) 372 { 373 pc->timestamp = timestamp; 374 } 375 376 void 377 pkg_create_set_overwrite(struct pkg_create *pc, bool overwrite) 378 { 379 pc->overwrite = overwrite; 380 } 381 382 static int 383 hash_file(struct pkg *pkg) 384 { 385 char hash_dest[MAXPATHLEN]; 386 char filename[MAXPATHLEN]; 387 388 /* Find the hash and rename the file and create a symlink */ 389 pkg_snprintf(filename, sizeof(filename), "%n-%v.pkg", 390 pkg, pkg); 391 pkg->sum = pkg_checksum_file(filename, 392 PKG_HASH_TYPE_SHA256_HEX); 393 pkg_snprintf(hash_dest, sizeof(hash_dest), "%n-%v-%z.pkg", 394 pkg, pkg, pkg); 395 396 pkg_debug(1, "Rename the pkg file from: %s to: %s", 397 filename, hash_dest); 398 if (rename(filename, hash_dest) == -1) { 399 pkg_emit_errno("rename", hash_dest); 400 unlink(hash_dest); 401 return (EPKG_FATAL); 402 } 403 if (symlink(hash_dest, filename) == -1) { 404 pkg_emit_errno("symlink", hash_dest); 405 return (EPKG_FATAL); 406 } 407 return (EPKG_OK); 408 } 409 410 int 411 pkg_create_i(struct pkg_create *pc, struct pkg *pkg, bool hash) 412 { 413 struct packing *pkg_archive = NULL; 414 int ret; 415 416 unsigned required_flags = PKG_LOAD_DEPS | PKG_LOAD_FILES | 417 PKG_LOAD_CATEGORIES | PKG_LOAD_DIRS | PKG_LOAD_SCRIPTS | 418 PKG_LOAD_OPTIONS | PKG_LOAD_LICENSES | PKG_LOAD_LUA_SCRIPTS; 419 420 assert(pkg->type == PKG_INSTALLED || pkg->type == PKG_OLD_FILE); 421 422 pkg_archive = pkg_create_archive(pkg, pc, required_flags); 423 if (pkg_archive == NULL) { 424 if (errno == EEXIST) 425 return (EPKG_EXIST); 426 pkg_emit_error("unable to create archive"); 427 return (EPKG_FATAL); 428 } 429 430 if ((ret = pkg_create_from_dir(pkg, NULL, pc, pkg_archive, false)) != EPKG_OK) { 431 pkg_emit_error("package creation failed"); 432 } 433 packing_finish(pkg_archive); 434 435 if (hash && ret == EPKG_OK) 436 ret = hash_file(pkg); 437 438 return (ret); 439 } 440 441 int 442 pkg_create(struct pkg_create *pc, const char *metadata, const char *plist, 443 bool hash) 444 { 445 struct pkg *pkg = NULL; 446 struct packing *pkg_archive = NULL; 447 int ret = EPKG_FATAL; 448 449 pkg_debug(1, "Creating package"); 450 if (pkg_new(&pkg, PKG_FILE) != EPKG_OK) { 451 return (EPKG_FATAL); 452 } 453 454 if (load_metadata(pkg, metadata, plist, pc->rootdir) != EPKG_OK) { 455 pkg_free(pkg); 456 return (EPKG_FATAL); 457 } 458 fixup_abi(pkg, pc->rootdir, false); 459 460 pkg_archive = pkg_create_archive(pkg, pc, 0); 461 if (pkg_archive == NULL) { 462 if (errno == EEXIST) { 463 pkg_emit_notice("%s-%s already packaged, skipping...\n", 464 pkg->name, pkg->version); 465 pkg_free(pkg); 466 return (EPKG_EXIST); 467 } 468 pkg_free(pkg); 469 return (EPKG_FATAL); 470 } 471 472 if ((ret = pkg_create_from_dir(pkg, pc->rootdir, pc, pkg_archive, true)) != EPKG_OK) 473 pkg_emit_error("package creation failed"); 474 475 packing_finish(pkg_archive); 476 if (hash && ret == EPKG_OK) 477 ret = hash_file(pkg); 478 479 pkg_free(pkg); 480 return (ret); 481 } 482 483 static int 484 pkg_load_message_from_file(int fd, struct pkg *pkg, const char *path) 485 { 486 char *buf = NULL; 487 off_t size = 0; 488 int ret; 489 ucl_object_t *obj; 490 491 assert(pkg != NULL); 492 assert(path != NULL); 493 494 if (faccessat(fd, path, F_OK, 0) == -1) { 495 return (EPKG_FATAL); 496 } 497 498 pkg_debug(1, "Reading message: '%s'", path); 499 if ((ret = file_to_bufferat(fd, path, &buf, &size)) != EPKG_OK) { 500 return (ret); 501 } 502 503 if (*buf == '[') { 504 ret = pkg_message_from_str(pkg, buf, size); 505 free(buf); 506 return (ret); 507 } 508 obj = ucl_object_fromstring_common(buf, size, 509 UCL_STRING_RAW|UCL_STRING_TRIM); 510 ret = pkg_message_from_ucl(pkg, obj); 511 ucl_object_unref(obj); 512 free(buf); 513 514 return (ret); 515 } 516 517 /* TODO use file descriptor for rootdir */ 518 static int 519 load_manifest(struct pkg *pkg, const char *metadata, const char *plist, 520 const char *rootdir) 521 { 522 int ret; 523 524 ret = pkg_parse_manifest_file(pkg, metadata); 525 526 if (ret == EPKG_OK && plist != NULL) 527 ret = ports_parse_plist(pkg, plist, rootdir); 528 return (ret); 529 } 530 531 /* TODO use file descriptor for rootdir */ 532 static int 533 load_metadata(struct pkg *pkg, const char *metadata, const char *plist, 534 const char *rootdir) 535 { 536 regex_t preg; 537 regmatch_t pmatch[2]; 538 size_t size; 539 int fd, i; 540 541 /* Let's see if we have a directory or a manifest */ 542 if ((fd = open(metadata, O_DIRECTORY|O_CLOEXEC)) == -1) { 543 if (errno == ENOTDIR) 544 return (load_manifest(pkg, metadata, plist, rootdir)); 545 pkg_emit_errno("open", metadata); 546 return (EPKG_FATAL); 547 } 548 549 if ((pkg_parse_manifest_fileat(fd, pkg, "+MANIFEST")) != EPKG_OK) { 550 pkg_emit_error("Error parsing %s/+MANIFEST", metadata); 551 close(fd); 552 return (EPKG_FATAL); 553 } 554 /* ensure the uid is properly */ 555 free(pkg->uid); 556 pkg->uid = xstrdup(pkg->name); 557 558 pkg_load_message_from_file(fd, pkg, "+DISPLAY"); 559 if (pkg->desc == NULL) 560 pkg_set_from_fileat(fd, pkg, PKG_ATTR_DESC, "+DESC", false); 561 562 for (i = 0; scripts[i] != NULL; i++) { 563 if (faccessat(fd, scripts[i], F_OK, 0) == 0) 564 pkg_addscript_fileat(fd, pkg, scripts[i]); 565 } 566 567 for (i = 0; lua_scripts[i] != NULL; i++) { 568 if (faccessat(fd, lua_scripts[i], F_OK, 0) == 0) 569 pkg_addluascript_fileat(fd, pkg, lua_scripts[i]); 570 } 571 572 if (plist != NULL && ports_parse_plist(pkg, plist, rootdir) != EPKG_OK) { 573 return (EPKG_FATAL); 574 } 575 close(fd); 576 577 if (pkg->www == NULL) { 578 if (pkg->desc == NULL) { 579 pkg_emit_error("No www or desc defined in manifest"); 580 return (EPKG_FATAL); 581 } 582 regcomp(&preg, "^WWW:[[:space:]]*(.*)$", 583 REG_EXTENDED|REG_ICASE|REG_NEWLINE); 584 if (regexec(&preg, pkg->desc, 2, pmatch, 0) == 0) { 585 size = pmatch[1].rm_eo - pmatch[1].rm_so; 586 pkg->www = xstrndup(&pkg->desc[pmatch[1].rm_so], size); 587 } else { 588 pkg->www = xstrdup("UNKNOWN"); 589 } 590 regfree(&preg); 591 } 592 593 return (EPKG_OK); 594 } 595 596 static void 597 fixup_abi(struct pkg *pkg, const char *rootdir, bool testing) 598 { 599 bool defaultarch = false; 600 601 /* if no arch autodetermine it */ 602 if (pkg->abi == NULL) { 603 if (ctx.abi.os == PKG_OS_FREEBSD) { 604 char *str_osversion; 605 xasprintf(&str_osversion, "%d", pkg_abi_get_freebsd_osversion(&ctx.abi)); 606 pkg_kv_add(&pkg->annotations, "FreeBSD_version", str_osversion, "annotation"); 607 } 608 pkg->abi = pkg_abi_to_string(&ctx.abi); 609 defaultarch = true; 610 } 611 612 if (!testing) 613 pkg_analyse_files(NULL, pkg, rootdir); 614 615 if (ctx.developer_mode) 616 suggest_arch(pkg, defaultarch); 617 } 618 619 int 620 pkg_load_metadata(struct pkg *pkg, const char *mfile, const char *md_dir, 621 const char *plist, const char *rootdir, bool testing) 622 { 623 int ret; 624 625 ret = load_metadata(pkg, md_dir != NULL ? md_dir: mfile, plist, rootdir); 626 if (ret != EPKG_OK) 627 return (ret); 628 629 fixup_abi(pkg, rootdir, testing); 630 return (ret); 631 } 632 633 static int64_t count; 634 static int64_t maxcount; 635 static const char *what; 636 637 static int magnitude(int64_t num) 638 { 639 int oom; 640 641 if (num == 0) 642 return (1); 643 if (num < 0) 644 num = -num; 645 646 for (oom = 1; num >= 10; oom++) 647 num /= 10; 648 649 return (oom); 650 } 651 652 static void 653 counter_init(const char *count_what, int64_t max) 654 { 655 count = 0; 656 what = count_what; 657 maxcount = max; 658 pkg_emit_progress_start("%-20s%*s[%jd]", what, 659 6 - magnitude(maxcount), " ", (intmax_t)maxcount); 660 661 return; 662 } 663 664 static void 665 counter_count(void) 666 { 667 count++; 668 669 if (count % TICK == 0) 670 pkg_emit_progress_tick(count, maxcount); 671 672 return; 673 } 674 675 static void 676 counter_end(void) 677 { 678 pkg_emit_progress_tick(count, maxcount); 679 return; 680 }