fetch_libfetch.c
1 /*- 2 * Copyright (c) 2020-2026 Baptiste Daroussin <bapt@FreeBSD.org> 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer 9 * in this position and unchanged. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 #include <sys/param.h> 27 #include <sys/wait.h> 28 #include <sys/socket.h> 29 #include <sys/time.h> 30 31 #include <ctype.h> 32 #include <fcntl.h> 33 #include <errno.h> 34 #include <stdio.h> 35 #include <string.h> 36 #include <fetch.h> 37 #include <paths.h> 38 #include <poll.h> 39 40 #include <bsd_compat.h> 41 42 #include "pkg.h" 43 #include "private/event.h" 44 #include "private/pkg.h" 45 #include "private/fetch.h" 46 #include "private/utils.h" 47 48 struct http_mirror { 49 struct url *url; 50 bool reldoc; 51 struct http_mirror *next; 52 }; 53 54 static void 55 gethttpmirrors(struct pkg_repo *repo, const char *url, bool withdoc) { 56 FILE *f; 57 char *line = NULL, *walk; 58 size_t linecap = 0; 59 ssize_t linelen; 60 struct http_mirror *m; 61 struct url *u; 62 63 if ((f = fetchGetURL(url, "")) == NULL) 64 return; 65 66 while ((linelen = getline(&line, &linecap, f)) > 0) { 67 if (strncmp(line, "URL:", 4) == 0) { 68 walk = line; 69 /* trim '\n' */ 70 if (walk[linelen - 1] == '\n') 71 walk[linelen - 1 ] = '\0'; 72 73 walk += 4; 74 while (isspace(*walk)) { 75 walk++; 76 } 77 if (*walk == '\0') 78 continue; 79 80 if ((u = fetchParseURL(walk)) != NULL) { 81 m = xmalloc(sizeof(struct http_mirror)); 82 m->reldoc = withdoc; 83 m->url = u; 84 m->next = NULL; 85 LL_APPEND(repo->http, m); 86 } 87 } 88 } 89 90 free(line); 91 fclose(f); 92 } 93 94 int 95 libfetch_open(struct pkg_repo *repo, struct fetch_item *fi) 96 { 97 struct url *u; 98 struct url *repourl; 99 int64_t max_retry, retry; 100 int64_t fetch_timeout; 101 char docpath[MAXPATHLEN]; 102 char zone[MAXHOSTNAMELEN + 24]; 103 char *doc, *reldoc, *opts; 104 struct dns_srvinfo *srv_current = NULL; 105 struct http_mirror *http_current = NULL; 106 struct url_stat st; 107 xstring *fetchOpts = NULL; 108 109 max_retry = pkg_object_int(pkg_config_get("FETCH_RETRY")); 110 fetch_timeout = pkg_object_int(pkg_config_get("FETCH_TIMEOUT")); 111 112 fetchTimeout = (int)MIN(fetch_timeout, INT_MAX); 113 if (fetch_timeout > 0) { 114 fetchSpeedLimit = 2 * 1024; /* 2KB/s, same as curl fetcher */ 115 fetchSpeedTime = (int)MIN(fetch_timeout, INT_MAX); 116 } 117 118 u = fetchParseURL(fi->url); 119 if (u == NULL) { 120 pkg_emit_error("%s: parse error", fi->url); 121 return (EPKG_FATAL); 122 } 123 124 repourl = fetchParseURL(repo->url); 125 if (repourl == NULL) { 126 pkg_emit_error("%s: parse error", repo->url); 127 fetchFreeURL(u); 128 return (EPKG_FATAL); 129 } 130 retry = max_retry; 131 doc = u->doc; 132 reldoc = doc + strlen(repourl->doc); 133 fetchFreeURL(repourl); 134 135 u->ims_time = fi->mtime; 136 if (fi->offset > 0) 137 u->offset = fi->offset; 138 139 /* HTTP authentication */ 140 const char *userpasswd = get_http_auth(); 141 if (userpasswd != NULL) { 142 const char *colon = strchr(userpasswd, ':'); 143 if (colon != NULL) { 144 size_t ulen = colon - userpasswd; 145 if (ulen < sizeof(u->user)) 146 strlcpy(u->user, userpasswd, ulen + 1); 147 strlcpy(u->pwd, colon + 1, sizeof(u->pwd)); 148 } 149 } 150 151 pkg_dbg(PKG_DBG_FETCH, 1, "libfetch> connecting"); 152 153 while (repo->fh == NULL) { 154 if (repo->mirror_type == SRV && 155 (strncmp(u->scheme, "http", 4) == 0)) { 156 if (repo->srv == NULL) { 157 snprintf(zone, sizeof(zone), 158 "_%s._tcp.%s", u->scheme, u->host); 159 repo->srv = dns_getsrvinfo(zone); 160 } 161 srv_current = repo->srv; 162 } else if (repo->mirror_type == HTTP && 163 strncmp(u->scheme, "http", 4) == 0) { 164 if (u->port == 0) { 165 if (strcmp(u->scheme, "https") == 0) 166 u->port = 443; 167 else 168 u->port = 80; 169 } 170 snprintf(zone, sizeof(zone), 171 "%s://%s:%d", u->scheme, u->host, u->port); 172 if (repo->http == NULL) 173 gethttpmirrors(repo, zone, false); 174 if (repo->http == NULL) 175 gethttpmirrors(repo, repo->url, true); 176 http_current = repo->http; 177 } 178 if (repo->mirror_type == SRV && repo->srv != NULL) { 179 strlcpy(u->host, srv_current->host, sizeof(u->host)); 180 u->port = srv_current->port; 181 } else if (repo->mirror_type == HTTP && 182 http_current != NULL) { 183 strlcpy(u->scheme, http_current->url->scheme, sizeof(u->scheme)); 184 strlcpy(u->host, http_current->url->host, sizeof(u->host)); 185 snprintf(docpath, sizeof(docpath), "%s%s", 186 http_current->url->doc, http_current->reldoc ? reldoc : doc); 187 u->doc = docpath; 188 u->port = http_current->url->port; 189 } 190 fetchOpts = xstring_new(); 191 fputs("i", fetchOpts->fp); 192 if (repo->ip == IPV4) 193 fputs("4", fetchOpts->fp); 194 else if (repo->ip == IPV6) 195 fputs("6", fetchOpts->fp); 196 197 if (ctx.debug_level >= 4) 198 fputs("v", fetchOpts->fp); 199 200 opts = xstring_get(fetchOpts); 201 pkg_dbg(PKG_DBG_FETCH, 1, 202 "libfetch> fetching from: %s://%s%s%s%s with opts \"%s\"", 203 u->scheme, 204 u->user, 205 u->user[0] != '\0' ? "@" : "", 206 u->host, 207 u->doc, 208 opts); 209 210 repo->fh = fetchXGet(u, &st, opts); 211 if (repo->fh == NULL) { 212 if (fetchLastErrCode == FETCH_OK) { 213 fetchFreeURL(u); 214 return (EPKG_UPTODATE); 215 } 216 if (fetchLastErrCode == FETCH_ABORT) { 217 fetchFreeURL(u); 218 return (EPKG_CANCEL); 219 } 220 if (fetchLastErrCode == FETCH_UNAVAIL) { 221 if (!repo->silent) 222 pkg_emit_error("%s://%s%s%s%s: %s", 223 u->scheme, 224 u->user, 225 u->user[0] != '\0' ? "@" : "", 226 u->host, 227 u->doc, 228 fetchLastErrString); 229 fetchFreeURL(u); 230 return (EPKG_ENOENT); 231 } 232 if (fetchLastErrCode == FETCH_NETWORK || 233 fetchLastErrCode == FETCH_RESOLV || 234 fetchLastErrCode == FETCH_DOWN) { 235 pkg_emit_pkg_errno(EPKG_NONETWORK, 236 "libfetch_open", NULL); 237 } 238 --retry; 239 if (retry <= 0) { 240 if (!repo->silent) 241 pkg_emit_error("%s://%s%s%s%s: %s", 242 u->scheme, 243 u->user, 244 u->user[0] != '\0' ? "@" : "", 245 u->host, 246 u->doc, 247 fetchLastErrString); 248 fetchFreeURL(u); 249 return (EPKG_FATAL); 250 } 251 if (repo->mirror_type == SRV && repo->srv != NULL) { 252 srv_current = srv_current->next; 253 if (srv_current == NULL) 254 srv_current = repo->srv; 255 } else if (repo->mirror_type == HTTP && 256 http_current != NULL) { 257 http_current = http_current->next; 258 if (http_current == NULL) 259 http_current = repo->http; 260 } 261 } 262 } 263 fi->size = st.size > 0 ? st.size : 0; 264 fi->mtime = st.mtime; 265 fetchFreeURL(u); 266 return (EPKG_OK); 267 } 268 269 int 270 libfetch_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi) 271 { 272 int ret; 273 274 ret = stdio_fetch(repo, dest, fi); 275 276 if (ret == EPKG_OK && ferror(repo->fh)) { 277 pkg_emit_error("%s: %s", fi->url, fetchLastErrString); 278 return (EPKG_FATAL); 279 } 280 return (ret); 281 } 282 283 void 284 libfetch_cleanup(struct pkg_repo *repo) 285 { 286 if (repo->mirror_type == HTTP) { 287 struct http_mirror *m, *tmp; 288 LL_FOREACH_SAFE(repo->http, m, tmp) { 289 fetchFreeURL(m->url); 290 free(m); 291 } 292 repo->http = NULL; 293 } else if (repo->mirror_type == SRV) { 294 struct dns_srvinfo *s, *stmp; 295 LL_FOREACH_SAFE(repo->srv, s, stmp) { 296 free(s); 297 } 298 repo->srv = NULL; 299 } 300 fh_close(repo); 301 }