guix-codesign
1 #!/usr/bin/env bash 2 export LC_ALL=C 3 set -e -o pipefail 4 5 # Source the common prelude, which: 6 # 1. Checks if we're at the top directory of the Bitcoin Core repository 7 # 2. Defines a few common functions and variables 8 # 9 # shellcheck source=libexec/prelude.bash 10 source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" 11 12 13 ################### 14 ## SANITY CHECKS ## 15 ################### 16 17 ################ 18 # Required non-builtin commands should be invocable 19 ################ 20 21 check_tools cat mkdir git guix 22 23 ################ 24 # Required env vars should be non-empty 25 ################ 26 27 cmd_usage() { 28 cat <<EOF 29 Synopsis: 30 31 env DETACHED_SIGS_REPO=<path/to/bitcoin-detached-sigs> \\ 32 ./contrib/guix/guix-codesign 33 34 EOF 35 } 36 37 if [ -z "$DETACHED_SIGS_REPO" ]; then 38 cmd_usage 39 exit 1 40 fi 41 42 ################ 43 # GUIX_BUILD_OPTIONS should be empty 44 ################ 45 # 46 # GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that 47 # can perform builds. This seems like what we want instead of 48 # ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually 49 # _appended_ to normal command-line options. Meaning that they will take 50 # precedence over the command-specific ADDITIONAL_GUIX_<CMD>_FLAGS. 51 # 52 # This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's 53 # existence here and direct users of this script to use our (more flexible) 54 # custom environment variables. 55 if [ -n "$GUIX_BUILD_OPTIONS" ]; then 56 cat << EOF 57 Error: Environment variable GUIX_BUILD_OPTIONS is not empty: 58 '$GUIX_BUILD_OPTIONS' 59 60 Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset 61 GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options 62 across guix commands or ADDITIONAL_GUIX_<CMD>_FLAGS to set build options for a 63 specific guix command. 64 65 See contrib/guix/README.md for more details. 66 EOF 67 exit 1 68 fi 69 70 ################ 71 # SOURCE_DATE_EPOCH should not unintentionally be set 72 ################ 73 74 check_source_date_epoch 75 76 ################ 77 # The codesignature git worktree should not be dirty 78 ################ 79 80 if ! git -C "$DETACHED_SIGS_REPO" diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then 81 cat << EOF 82 ERR: The DETACHED CODESIGNATURE git worktree is dirty, which may lead to broken builds. 83 84 Aborting... 85 86 Hint: To make your git worktree clean, You may want to: 87 1. Commit your changes, 88 2. Stash your changes, or 89 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on 90 using a dirty worktree 91 EOF 92 exit 1 93 fi 94 95 ################ 96 # Build directories should not exist 97 ################ 98 99 # Default to building for all supported HOSTs (overridable by environment) 100 export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" 101 102 # Usage: distsrc_for_host HOST 103 # 104 # HOST: The current platform triple we're building for 105 # 106 distsrc_for_host() { 107 echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}-codesigned" 108 } 109 110 # Accumulate a list of build directories that already exist... 111 hosts_distsrc_exists="" 112 for host in $HOSTS; do 113 if [ -e "$(distsrc_for_host "$host")" ]; then 114 hosts_distsrc_exists+=" ${host}" 115 fi 116 done 117 118 if [ -n "$hosts_distsrc_exists" ]; then 119 # ...so that we can print them out nicely in an error message 120 cat << EOF 121 ERR: Build directories for this commit already exist for the following platform 122 triples you're attempting to build, probably because of previous builds. 123 Please remove, or otherwise deal with them prior to starting another build. 124 125 Aborting... 126 127 Hint: To blow everything away, you may want to use: 128 129 $ ./contrib/guix/guix-clean 130 131 Specifically, this will remove all files without an entry in the index, 132 excluding the SDK directory, the depends download cache, the depends built 133 packages cache, the garbage collector roots for Guix environments, and the 134 output directory. 135 EOF 136 for host in $hosts_distsrc_exists; do 137 echo " ${host} '$(distsrc_for_host "$host")'" 138 done 139 exit 1 140 else 141 mkdir -p "$DISTSRC_BASE" 142 fi 143 144 145 ################ 146 # Codesigning tarballs SHOULD exist 147 ################ 148 149 # Usage: outdir_for_host HOST SUFFIX 150 # 151 # HOST: The current platform triple we're building for 152 # 153 outdir_for_host() { 154 echo "${OUTDIR_BASE}/${1}${2:+-${2}}" 155 } 156 157 158 codesigning_tarball_for_host() { 159 case "$1" in 160 *mingw*) 161 echo "$(outdir_for_host "$1")/${DISTNAME}-win64-codesigning.tar.gz" 162 ;; 163 *darwin*) 164 echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-codesigning.tar.gz" 165 ;; 166 *) 167 exit 1 168 ;; 169 esac 170 } 171 172 # Accumulate a list of build directories that already exist... 173 hosts_codesigning_tarball_missing="" 174 for host in $HOSTS; do 175 if [ ! -e "$(codesigning_tarball_for_host "$host")" ]; then 176 hosts_codesigning_tarball_missing+=" ${host}" 177 fi 178 done 179 180 if [ -n "$hosts_codesigning_tarball_missing" ]; then 181 # ...so that we can print them out nicely in an error message 182 cat << EOF 183 ERR: Codesigning tarballs do not exist 184 ... 185 186 EOF 187 for host in $hosts_codesigning_tarball_missing; do 188 echo " ${host} '$(codesigning_tarball_for_host "$host")'" 189 done 190 exit 1 191 fi 192 193 ################ 194 # Check that we can connect to the guix-daemon 195 ################ 196 197 cat << EOF 198 Checking that we can connect to the guix-daemon... 199 200 Hint: If this hangs, you may want to try turning your guix-daemon off and on 201 again. 202 203 EOF 204 if ! guix gc --list-failures > /dev/null; then 205 cat << EOF 206 207 ERR: Failed to connect to the guix-daemon, please ensure that one is running and 208 reachable. 209 EOF 210 exit 1 211 fi 212 213 # Developer note: we could use `guix repl` for this check and run: 214 # 215 # (import (guix store)) (close-connection (open-connection)) 216 # 217 # However, the internal API is likely to change more than the CLI invocation 218 219 220 ######### 221 # SETUP # 222 ######### 223 224 # Determine the maximum number of jobs to run simultaneously (overridable by 225 # environment) 226 JOBS="${JOBS:-$(nproc)}" 227 228 # Determine the reference time used for determinism (overridable by environment) 229 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" 230 231 # Make sure an output directory exists for our builds 232 OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" 233 mkdir -p "$OUTDIR_BASE" 234 235 # Usage: profiledir_for_host HOST SUFFIX 236 # 237 # HOST: The current platform triple we're building for 238 # 239 profiledir_for_host() { 240 echo "${PROFILES_BASE}/${1}${2:+-${2}}" 241 } 242 243 ######### 244 # BUILD # 245 ######### 246 247 # Function to be called when codesigning for host ${1} and the user interrupts 248 # the codesign 249 int_trap() { 250 cat << EOF 251 ** INT received while codesigning ${1}, you may want to clean up the relevant 252 work directories (e.g. distsrc-*) before recodesigning 253 254 Hint: To blow everything away, you may want to use: 255 256 $ ./contrib/guix/guix-clean 257 258 Specifically, this will remove all files without an entry in the index, 259 excluding the SDK directory, the depends download cache, the depends built 260 packages cache, the garbage collector roots for Guix environments, and the 261 output directory. 262 EOF 263 } 264 265 # Deterministically build Bitcoin Core 266 # shellcheck disable=SC2153 267 for host in $HOSTS; do 268 269 # Display proper warning when the user interrupts the build 270 trap 'int_trap ${host}' INT 271 272 ( 273 # Required for 'contrib/guix/manifest.scm' to output the right manifest 274 # for the particular $HOST we're building for 275 export HOST="$host" 276 277 # shellcheck disable=SC2030 278 cat << EOF 279 INFO: Codesigning ${VERSION:?not set} for platform triple ${HOST:?not set}: 280 ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} 281 ...from worktree directory: '${PWD}' 282 ...bind-mounted in container to: '/bitcoin' 283 ...in build directory: '$(distsrc_for_host "$HOST")' 284 ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' 285 ...outputting in: '$(outdir_for_host "$HOST" codesigned)' 286 ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)' 287 ...using detached signatures in: '${DETACHED_SIGS_REPO:?not set}' 288 ...bind-mounted in container to: '/detached-sigs' 289 EOF 290 291 292 # Run the build script 'contrib/guix/libexec/codesign.sh' in the build 293 # container specified by 'contrib/guix/manifest.scm'. 294 # 295 # Explanation of `guix shell` flags: 296 # 297 # --container run command within an isolated container 298 # 299 # Running in an isolated container minimizes build-time differences 300 # between machines and improves reproducibility 301 # 302 # --writable-root make the root filesystem writable 303 # 304 # --pure unset existing environment variables 305 # 306 # Same rationale as --container 307 # 308 # --no-cwd do not share current working directory with an 309 # isolated container 310 # 311 # When --container is specified, the default behavior is to share 312 # the current working directory with the isolated container at the 313 # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to 314 # '/home/satoshi/bitcoin/'). This means that the $PWD inside the 315 # container becomes a source of irreproducibility. --no-cwd disables 316 # this behaviour. 317 # 318 # --share=SPEC for containers, share writable host file system 319 # according to SPEC 320 # 321 # --share="$PWD"=/bitcoin 322 # 323 # maps our current working directory to /bitcoin 324 # inside the isolated container, which we later cd 325 # into. 326 # 327 # While we don't want to map our current working directory to the 328 # same exact path (as this introduces irreproducibility), we do want 329 # it to be at a _fixed_ path _somewhere_ inside the isolated 330 # container so that we have something to build. '/bitcoin' was 331 # chosen arbitrarily. 332 # 333 # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} 334 # 335 # fetch substitute from SUBSTITUTE_URLS if they are 336 # authorized 337 # 338 # Depending on the user's security model, it may be desirable to use 339 # substitutes (pre-built packages) from servers that the user trusts. 340 # Please read the README.md in the same directory as this file for 341 # more information. 342 # 343 # shellcheck disable=SC2086,SC2031 344 time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ 345 --container \ 346 --writable-root \ 347 --pure \ 348 --no-cwd \ 349 --share="$PWD"=/bitcoin \ 350 --share="$DISTSRC_BASE"=/distsrc-base \ 351 --share="$OUTDIR_BASE"=/outdir-base \ 352 --share="$DETACHED_SIGS_REPO"=/detached-sigs \ 353 --expose="$(git rev-parse --git-common-dir)" \ 354 --expose="$(git -C "$DETACHED_SIGS_REPO" rev-parse --git-common-dir)" \ 355 --cores="$JOBS" \ 356 --keep-failed \ 357 --fallback \ 358 --link-profile \ 359 --root="$(profiledir_for_host "${HOST}" codesigned)" \ 360 ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ 361 ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ 362 -- env HOST="$host" \ 363 DISTNAME="$DISTNAME" \ 364 JOBS="$JOBS" \ 365 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ 366 ${V:+V=1} \ 367 DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ 368 OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)" \ 369 DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ 370 DETACHED_SIGS_REPO=/detached-sigs \ 371 CODESIGNING_TARBALL="$(OUTDIR_BASE=/outdir-base && codesigning_tarball_for_host "$HOST")" \ 372 bash -c "cd /bitcoin && bash contrib/guix/libexec/codesign.sh" 373 ) 374 375 done