fetch.c
1 /*- 2 * Copyright (c) 2012-2023 Baptiste Daroussin <bapt@FreeBSD.org> 3 * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org> 4 * Copyright (c) 2014 Vsevolod Stakhov <vsevolod@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer 12 * in this position and unchanged. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/wait.h> 31 #include <sys/socket.h> 32 #include <sys/time.h> 33 34 #include <ctype.h> 35 #include <fcntl.h> 36 #include <errno.h> 37 #include <stdio.h> 38 #include <string.h> 39 #include <paths.h> 40 #include <poll.h> 41 42 #include <bsd_compat.h> 43 44 #include "pkg.h" 45 #include "private/event.h" 46 #include "private/pkg.h" 47 #include "private/utils.h" 48 #include "private/fetch.h" 49 50 static struct fetcher fetchers[] = { 51 { 52 .scheme = "tcp", 53 .open = tcp_open, 54 .close = NULL, 55 .cleanup = fh_close, 56 .fetch = stdio_fetch, 57 }, 58 { 59 .scheme = "ssh", 60 .open = ssh_open, 61 .close = NULL, 62 .cleanup = fh_close, 63 .fetch = stdio_fetch, 64 }, 65 { 66 .scheme = "pkg+https", 67 .open = libfetch_open, 68 .close = fh_close, 69 .cleanup = libfetch_cleanup, 70 .fetch = libfetch_fetch, 71 }, 72 { 73 .scheme = "pkg+http", 74 .open = libfetch_open, 75 .close = fh_close, 76 .cleanup = libfetch_cleanup, 77 .fetch = libfetch_fetch, 78 }, 79 { 80 .scheme = "https", 81 .open = libfetch_open, 82 .close = fh_close, 83 .cleanup = libfetch_cleanup, 84 .fetch = libfetch_fetch, 85 }, 86 { 87 .scheme = "http", 88 .open = libfetch_open, 89 .close = fh_close, 90 .cleanup = libfetch_cleanup, 91 .fetch = libfetch_fetch, 92 }, 93 { 94 .scheme = "file", 95 .open = file_open, 96 .close = fh_close, 97 .cleanup = NULL, 98 .fetch = stdio_fetch, 99 }, 100 }; 101 102 int 103 pkg_fetch_file_tmp(struct pkg_repo *repo, const char *url, char *dest, 104 time_t t, int *outfd) 105 { 106 int fd = -1; 107 int retcode = EPKG_FATAL; 108 struct fetch_item fi; 109 110 memset(&fi, 0, sizeof(struct fetch_item)); 111 112 fd = mkstemp(dest); 113 114 if (fd == -1) { 115 pkg_emit_errno("mkstemp", dest); 116 return(EPKG_FATAL); 117 } 118 119 /* Unlink immediately so the temp file is cleaned up even if the 120 * process is killed by a signal during the fetch (issue #1988) */ 121 unlink(dest); 122 123 fi.url = url; 124 fi.mtime = t; 125 retcode = pkg_fetch_file_to_fd(repo, fd, &fi, false); 126 127 if (fi.mtime != 0) { 128 struct timespec ts[2] = { 129 { 130 .tv_sec = fi.mtime, 131 .tv_nsec = 0 132 }, 133 { 134 .tv_sec = fi.mtime, 135 .tv_nsec = 0 136 } 137 }; 138 futimens(fd, ts); 139 } 140 141 if (retcode != EPKG_OK) { 142 close(fd); 143 return (retcode); 144 } 145 146 /* Rewind so the caller can read from the beginning */ 147 lseek(fd, 0, SEEK_SET); 148 *outfd = fd; 149 150 return (retcode); 151 } 152 153 int 154 pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t, 155 ssize_t offset, int64_t size) 156 { 157 int fd = -1; 158 int retcode = EPKG_FATAL; 159 struct fetch_item fi; 160 char *url_to_free = NULL; 161 162 fd = open(dest, O_CREAT|O_APPEND|O_WRONLY, 00644); 163 if (fd == -1) { 164 pkg_emit_errno("open", dest); 165 return(EPKG_FATAL); 166 } 167 168 if (repo != NULL) { 169 xasprintf(&url_to_free, "%s/%s", repo->url, url); 170 fi.url = url_to_free; 171 } else { 172 fi.url = url; 173 } 174 175 fi.offset = offset; 176 fi.size = size; 177 fi.mtime = t; 178 179 retcode = pkg_fetch_file_to_fd(repo, fd, &fi, false); 180 free(url_to_free); 181 182 if (t != 0) { 183 struct timespec ts[2] = { 184 { 185 .tv_sec = t, 186 .tv_nsec = 0 187 }, 188 { 189 .tv_sec = t, 190 .tv_nsec = 0 191 } 192 }; 193 futimens(fd, ts); 194 } 195 close(fd); 196 197 /* Remove local file if fetch failed */ 198 if (retcode != EPKG_OK) 199 unlink(dest); 200 201 return (retcode); 202 } 203 204 #define URL_SCHEME_PREFIX "pkg+" 205 206 static const struct fetcher * 207 select_fetcher(const char *url) 208 { 209 struct fetcher *f; 210 size_t nsz; 211 212 for (size_t i = 0; i < NELEM(fetchers); i++) { 213 nsz = strlen(fetchers[i].scheme); 214 215 if ((strncasecmp(url, fetchers[i].scheme, nsz) == 0) && 216 url[nsz] == ':') { 217 f = &fetchers[i]; 218 f->timeout = 219 pkg_object_int(pkg_config_get("FETCH_TIMEOUT")); 220 return (f); 221 } 222 } 223 return (NULL); 224 225 } 226 int 227 pkg_fetch_file_to_fd(struct pkg_repo *repo, int dest, struct fetch_item *fi, 228 bool silent) 229 { 230 struct pkg_kv *kv; 231 kvlist_t envtorestore = vec_init(); 232 c_charv_t envtounset = vec_init(); 233 char *tmp; 234 int retcode = EPKG_OK; 235 struct pkg_repo *fakerepo = NULL; 236 size_t nsz; 237 238 /* A URL of the form http://host.example.com/ where 239 * host.example.com does not resolve as a simple A record is 240 * not valid according to RFC 2616 Section 3.2.2. Our usage 241 * with SRV records is incorrect. However it is encoded into 242 * /usr/sbin/pkg in various releases so we can't just drop it. 243 * 244 * Instead, introduce new pkg+http://, pkg+https://, 245 * pkg+ssh://, pkg+file:// to support the 246 * SRV-style server discovery, and also to allow eg. Firefox 247 * to run pkg-related stuff given a pkg+foo:// URL. 248 * 249 * Error if using plain http://, https:// etc with SRV 250 */ 251 252 pkg_dbg(PKG_DBG_FETCH, 1, "Request to fetch %s", fi->url); 253 if (repo == NULL) { 254 fakerepo = xcalloc(1, sizeof(struct pkg_repo)); 255 fakerepo->url = xstrdup(fi->url); 256 fakerepo->mirror_type = NOMIRROR; 257 repo = fakerepo; 258 } 259 260 if (repo->fetcher == NULL) 261 repo->fetcher = select_fetcher(fi->url); 262 if (repo->fetcher == NULL) { 263 pkg_emit_error("Unknown scheme: %s", fi->url); 264 return (EPKG_FATAL); 265 } 266 267 nsz = strlen(URL_SCHEME_PREFIX); 268 if (strncasecmp(URL_SCHEME_PREFIX, fi->url, nsz) == 0) { 269 if (repo->mirror_type != SRV) { 270 pkg_emit_error("packagesite URL error for %s -- " 271 URL_SCHEME_PREFIX 272 ":// implies SRV mirror type", fi->url); 273 274 /* Too early for there to be anything to cleanup */ 275 return(EPKG_FATAL); 276 } 277 fi->url += nsz; 278 } 279 280 repo->silent = silent; 281 vec_foreach(repo->env, i) { 282 if ((tmp = getenv(repo->env.d[i]->key)) != NULL) { 283 kv = xcalloc(1, sizeof(*kv)); 284 kv->key = xstrdup(repo->env.d[i]->key); 285 kv->value = xstrdup(tmp); 286 vec_push(&envtorestore, kv); 287 } else { 288 vec_push(&envtounset, repo->env.d[i]->key); 289 } 290 setenv(repo->env.d[i]->key, repo->env.d[i]->value, 1); 291 } 292 293 if ((retcode = repo->fetcher->open(repo, fi)) != EPKG_OK) 294 goto cleanup; 295 pkg_dbg(PKG_DBG_FETCH, 1, "Fetch: fetcher used: %s", repo->fetcher->scheme); 296 297 retcode = repo->fetcher->fetch(repo, dest, fi); 298 if (retcode == EPKG_OK) 299 pkg_emit_fetch_finished(fi->url); 300 301 cleanup: 302 vec_foreach(envtorestore, i) { 303 setenv(envtorestore.d[i]->key, envtorestore.d[i]->value, 1); 304 vec_remove_and_free(&envtorestore, i, pkg_kv_free); 305 } 306 vec_free(&envtorestore); 307 while (vec_len(&envtounset) > 0) 308 unsetenv(vec_pop(&envtounset)); 309 vec_free(&envtounset); 310 311 if (repo->fetcher != NULL && repo->fetcher->close != NULL) 312 repo->fetcher->close(repo); 313 free(fakerepo); 314 315 if (retcode == EPKG_OK) { 316 struct timespec ts[2] = { 317 { 318 .tv_sec = fi->mtime, 319 .tv_nsec = 0 320 }, 321 { 322 .tv_sec = fi->mtime, 323 .tv_nsec = 0 324 } 325 }; 326 futimens(dest, ts); 327 } 328 329 return (retcode); 330 }