get_previous_releases.py
1 #!/usr/bin/env python3 2 # 3 # Copyright (c) 2018-present The Bitcoin Core developers 4 # Distributed under the MIT software license, see the accompanying 5 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 6 7 import argparse 8 import contextlib 9 from fnmatch import fnmatch 10 import hashlib 11 import os 12 from pathlib import Path 13 import platform 14 import re 15 import shutil 16 import subprocess 17 import sys 18 import time 19 import zipfile 20 21 sys.path.append(str(Path(__file__).resolve().parent)) 22 from download_utils import download_from_url 23 24 TAR = os.getenv('TAR', 'tar') 25 26 SHA256_SUMS = { 27 "0e2819135366f150d9906e294b61dff58fd1996ebd26c2f8e979d6c0b7a79580": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-aarch64-linux-gnu.tar.gz"}, 28 "d86fc90824a85c38b25c8488115178d5785dbc975f5ff674f9f5716bc8ad6e65": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-arm-linux-gnueabihf.tar.gz"}, 29 "1b0a7408c050e3d09a8be8e21e183ef7ee570385dc41216698cc3ab392a484e7": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-osx64.tar.gz"}, 30 "706e0472dbc933ed2757650d54cbcd780fd3829ebf8f609b32780c7eedebdbc9": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-x86_64-linux-gnu.tar.gz"}, 31 "5ea84b6a4c1b03d5f4d1a718fbed215e2fd0e66ca7d59ca6141218d163ce1301": {"tag": "v0.14.3", "archive": "bitcoin-0.14.3-win64.zip"}, 32 33 "60c93e3462c303eb080be7cf623f1a7684b37fd47a018ad3848bc23e13c84e1c": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-aarch64-linux-gnu.tar.gz"}, 34 "55b577e0fb306fb429d4be6c9316607753e8543e5946b542d75d876a2f08654c": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-arm-linux-gnueabihf.tar.gz"}, 35 "b9024dde373ea7dad707363e07ec7e265383204127539ae0c234bff3a61da0d1": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-osx64.tar.gz"}, 36 "fa71cb52ee5e0459cbf5248cdec72df27995840c796f58b304607a1ed4c165af": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-riscv64-linux-gnu.tar.gz"}, 37 "376194f06596ecfa40331167c39bc70c355f960280bd2a645fdbf18f66527397": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-x86_64-linux-gnu.tar.gz"}, 38 "e59fba67afce011d32b5d723a3a0be12da1b8a34f5d7966e504520c48d64716d": {"tag": "v0.20.1", "archive": "bitcoin-0.20.1-win64.zip"}, 39 40 "43416854330914992bbba2d0e9adf2a6fff4130be9af8ae2ef1186e743d9a3fe": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-aarch64-linux-gnu.tar.gz"}, 41 "f028af308eda45a3c4c90f9332f96b075bf21e3495c945ebce48597151808176": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-arm-linux-gnueabihf.tar.gz"}, 42 "695fb624fa6423f5da4f443b60763dd1d77488bfe5ef63760904a7b54e91298d": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-osx64.tar.gz"}, 43 "f8b2adfeae021a672effbc7bd40d5c48d6b94e53b2dd660f787340bf1a52e4e9": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-riscv64-linux-gnu.tar.gz"}, 44 "da7766775e3f9c98d7a9145429f2be8297c2672fe5b118fd3dc2411fb48e0032": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-x86_64-linux-gnu.tar.gz"}, 45 "1d0052c4ce80227fb6d0bc1c4e673ba21033e219c1f935d25f130ef7f43360d4": {"tag": "v0.21.0", "archive": "bitcoin-0.21.0-win64.zip"}, 46 47 "ac718fed08570a81b3587587872ad85a25173afa5f9fbbd0c03ba4d1714cfa3e": {"tag": "v22.0", "archive": "bitcoin-22.0-aarch64-linux-gnu.tar.gz"}, 48 "b8713c6c5f03f5258b54e9f436e2ed6d85449aa24c2c9972f91963d413e86311": {"tag": "v22.0", "archive": "bitcoin-22.0-arm-linux-gnueabihf.tar.gz"}, 49 "2744d199c3343b2d94faffdfb2c94d75a630ba27301a70e47b0ad30a7e0155e9": {"tag": "v22.0", "archive": "bitcoin-22.0-osx64.tar.gz"}, 50 "2cca5f99007d060aca9d8c7cbd035dfe2f040dd8200b210ce32cdf858479f70d": {"tag": "v22.0", "archive": "bitcoin-22.0-powerpc64-linux-gnu.tar.gz"}, 51 "91b1e012975c5a363b5b5fcc81b5b7495e86ff703ec8262d4b9afcfec633c30d": {"tag": "v22.0", "archive": "bitcoin-22.0-powerpc64le-linux-gnu.tar.gz"}, 52 "9cc3a62c469fe57e11485fdd32c916f10ce7a2899299855a2e479256ff49ff3c": {"tag": "v22.0", "archive": "bitcoin-22.0-riscv64-linux-gnu.tar.gz"}, 53 "59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16": {"tag": "v22.0", "archive": "bitcoin-22.0-x86_64-linux-gnu.tar.gz"}, 54 "9485e4b52ed6cebfe474ab4d7d0c1be6d0bb879ba7246a8239326b2230a77eb1": {"tag": "v22.0", "archive": "bitcoin-22.0-win64.zip"}, 55 56 "06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb": {"tag": "v23.0", "archive": "bitcoin-23.0-aarch64-linux-gnu.tar.gz"}, 57 "952c574366aff76f6d6ad1c9ee45a361d64fa04155e973e926dfe7e26f9703a3": {"tag": "v23.0", "archive": "bitcoin-23.0-arm-linux-gnueabihf.tar.gz"}, 58 "7c8bc63731aa872b7b334a8a7d96e33536ad77d49029bad179b09dca32cd77ac": {"tag": "v23.0", "archive": "bitcoin-23.0-arm64-apple-darwin.tar.gz"}, 59 "2caa5898399e415f61d9af80a366a3008e5856efa15aaff74b88acf429674c99": {"tag": "v23.0", "archive": "bitcoin-23.0-powerpc64-linux-gnu.tar.gz"}, 60 "217dd0469d0f4962d22818c368358575f6a0abcba8804807bb75325eb2f28b19": {"tag": "v23.0", "archive": "bitcoin-23.0-powerpc64le-linux-gnu.tar.gz"}, 61 "078f96b1e92895009c798ab827fb3fde5f6719eee886bd0c0e93acab18ea4865": {"tag": "v23.0", "archive": "bitcoin-23.0-riscv64-linux-gnu.tar.gz"}, 62 "c816780583009a9dad426dc0c183c89be9da98906e1e2c7ebae91041c1aaaaf3": {"tag": "v23.0", "archive": "bitcoin-23.0-x86_64-apple-darwin.tar.gz"}, 63 "2cca490c1f2842884a3c5b0606f179f9f937177da4eadd628e3f7fd7e25d26d0": {"tag": "v23.0", "archive": "bitcoin-23.0-x86_64-linux-gnu.tar.gz"}, 64 "004b2e25b21e0f14cbcce6acec37f221447abbb3ea7931c689e508054bfc6cf6": {"tag": "v23.0", "archive": "bitcoin-23.0-win64.zip"}, 65 66 "0b48b9e69b30037b41a1e6b78fb7cbcc48c7ad627908c99686e81f3802454609": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-aarch64-linux-gnu.tar.gz"}, 67 "37d7660f0277301744e96426bbb001d2206b8d4505385dfdeedf50c09aaaef60": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-arm-linux-gnueabihf.tar.gz"}, 68 "90ed59e86bfda1256f4b4cad8cc1dd77ee0efec2492bcb5af61402709288b62c": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-arm64-apple-darwin.tar.gz"}, 69 "7590645e8676f8b5fda62dc20174474c4ac8fd0defc83a19ed908ebf2e94dc11": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-powerpc64-linux-gnu.tar.gz"}, 70 "79e89a101f23ff87816675b98769cd1ee91059f95c5277f38f48f21a9f7f8509": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-powerpc64le-linux-gnu.tar.gz"}, 71 "6b163cef7de4beb07b8cb3347095e0d76a584019b1891135cd1268a1f05b9d88": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-riscv64-linux-gnu.tar.gz"}, 72 "e2f751512f3c0f00eb68ba946d9c829e6cf99422a61e8f5e0a7c109c318674d0": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-x86_64-apple-darwin.tar.gz"}, 73 "49df6e444515d457ea0b885d66f521f2a26ca92ccf73d5296082e633544253bf": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-x86_64-linux-gnu.tar.gz"}, 74 "8784ce304f22c495392d3adfd7fc2c645d093db9bd4d42666c41adf540539fff": {"tag": "v24.0.1", "archive": "bitcoin-24.0.1-win64.zip"}, 75 76 "3a7bdd959a0b426624f63f394f25e5b7769a5a2f96f8126dcc2ea53f3fa5212b": {"tag": "v25.0", "archive": "bitcoin-25.0-aarch64-linux-gnu.tar.gz"}, 77 "e537c8630b05e63242d979c3004f851fd73c2a10b5b4fdbb161788427c7b3c0f": {"tag": "v25.0", "archive": "bitcoin-25.0-arm-linux-gnueabihf.tar.gz"}, 78 "3b35075d6c1209743611c705a13575be2668bc069bc6301ce78a2e1e53ebe7cc": {"tag": "v25.0", "archive": "bitcoin-25.0-arm64-apple-darwin.tar.gz"}, 79 "0c8e135a6fd297270d3b65196042d761453493a022b5ff7fb847fc911e938214": {"tag": "v25.0", "archive": "bitcoin-25.0-powerpc64-linux-gnu.tar.gz"}, 80 "fa8af160782f5adfcea570f72b947073c1663b3e9c3cd0f82b216b609fe47573": {"tag": "v25.0", "archive": "bitcoin-25.0-powerpc64le-linux-gnu.tar.gz"}, 81 "fe6e347a66043946920c72c9c4afca301968101e6b82fb90a63d7885ebcceb32": {"tag": "v25.0", "archive": "bitcoin-25.0-riscv64-linux-gnu.tar.gz"}, 82 "5708fc639cdfc27347cccfd50db9b73b53647b36fb5f3a4a93537cbe8828c27f": {"tag": "v25.0", "archive": "bitcoin-25.0-x86_64-apple-darwin.tar.gz"}, 83 "33930d432593e49d58a9bff4c30078823e9af5d98594d2935862788ce8a20aec": {"tag": "v25.0", "archive": "bitcoin-25.0-x86_64-linux-gnu.tar.gz"}, 84 "7154b35ecc8247589070ae739b7c73c4dee4794bea49eb18dc66faed65b819e7": {"tag": "v25.0", "archive": "bitcoin-25.0-win64.zip"}, 85 86 "10581b6d4aeb091c08c9e69eb4e4ced000038c009286dc1edb8a876656ccf6df": {"tag": "v28.2", "archive": "bitcoin-28.2-aarch64-linux-gnu.tar.gz"}, 87 "c08135c249688438b1ed6f48a2f192ffaae7c7111ffee5cfad59fdae355bb47f": {"tag": "v28.2", "archive": "bitcoin-28.2-arm-linux-gnueabihf.tar.gz"}, 88 "c0270ed50effc174f7ff3332dba5183a8693999dac2ba78b37d8c8797b3ea2b2": {"tag": "v28.2", "archive": "bitcoin-28.2-arm64-apple-darwin.tar.gz"}, 89 "21c54bc7520e873c8b52c817af257a8a4aee15d81bb2492e5bd51af055ae4469": {"tag": "v28.2", "archive": "bitcoin-28.2-powerpc64-linux-gnu.tar.gz"}, 90 "8ad2bf5dd0a7fd04afa9cd2b1309621f8662f00c799f1971cd44ba9eb9d18ef1": {"tag": "v28.2", "archive": "bitcoin-28.2-riscv64-linux-gnu.tar.gz"}, 91 "866a4b703a2095301151c17dcc753e19e4dba61ec68d19709ec4f81ff4320103": {"tag": "v28.2", "archive": "bitcoin-28.2-x86_64-apple-darwin.tar.gz"}, 92 "98add5f220c01b387343b70edeb6273403fe081e22cd85fda132704cdcaa98aa": {"tag": "v28.2", "archive": "bitcoin-28.2-x86_64-linux-gnu.tar.gz"}, 93 "da0869639c323bbf6f264f1829083b9514e10179b90c34b09d8cbcab8a1897e3": {"tag": "v28.2", "archive": "bitcoin-28.2-win64.zip"}, 94 } 95 96 97 @contextlib.contextmanager 98 def pushd(new_dir) -> None: 99 previous_dir = os.getcwd() 100 os.chdir(new_dir) 101 try: 102 yield 103 finally: 104 os.chdir(previous_dir) 105 106 107 def download_binary(tag, args) -> int: 108 if Path(tag).is_dir(): 109 if not args.remove_dir: 110 print('Using cached {}'.format(tag)) 111 return 0 112 shutil.rmtree(tag) 113 114 bin_path = 'bin/bitcoin-core-{}'.format(tag[1:]) 115 116 match = re.compile('v(.*)(rc[0-9]+)$').search(tag) 117 if match: 118 bin_path = 'bin/bitcoin-core-{}/test.{}'.format( 119 match.group(1), match.group(2)) 120 121 host = args.host 122 if tag < "v23" and host in ["x86_64-apple-darwin", "arm64-apple-darwin"]: 123 host = "osx64" 124 125 archive_format = 'tar.gz' 126 if host == 'win64': 127 archive_format = 'zip' 128 129 archive = f'bitcoin-{tag[1:]}-{host}.{archive_format}' 130 archive_url = f'https://bitcoincore.org/{bin_path}/{archive}' 131 132 try: 133 download_from_url(archive_url, archive) 134 except Exception as e: 135 print(f"\nDownload failed: {e}", file=sys.stderr) 136 print("Retrying download after failure ...", file=sys.stderr) 137 time.sleep(12) 138 try: 139 download_from_url(archive_url, archive) 140 except Exception as e2: 141 print(f"\nDownload failed a second time: {e2}", file=sys.stderr) 142 return 1 143 144 hasher = hashlib.sha256() 145 with open(archive, "rb") as afile: 146 hasher.update(afile.read()) 147 archiveHash = hasher.hexdigest() 148 149 if archiveHash not in SHA256_SUMS or SHA256_SUMS[archiveHash]['archive'] != archive: 150 if archive in [v['archive'] for v in SHA256_SUMS.values()]: 151 print(f"Checksum {archiveHash} did not match", file=sys.stderr) 152 else: 153 print("Checksum for given version doesn't exist", file=sys.stderr) 154 return 1 155 156 print("Checksum matched") 157 Path(tag).mkdir() 158 159 # Extract archive 160 if host == 'win64': 161 try: 162 with zipfile.ZipFile(archive, 'r') as zip: 163 zip.extractall(tag) 164 # Remove the top level directory to match tar's --strip-components=1 165 extracted_items = os.listdir(tag) 166 top_level_dir = os.path.join(tag, extracted_items[0]) 167 # Move all files & subdirectories up one level 168 for item in os.listdir(top_level_dir): 169 shutil.move(os.path.join(top_level_dir, item), tag) 170 # Remove the now-empty top-level directory 171 os.rmdir(top_level_dir) 172 except Exception as e: 173 print(f"Zip extraction failed: {e}", file=sys.stderr) 174 return 1 175 else: 176 ret = subprocess.run([TAR, '-zxf', archive, '-C', tag, 177 '--strip-components=1', 178 'bitcoin-{tag}'.format(tag=tag[1:])]).returncode 179 if ret != 0: 180 print(f"Failed to extract the {tag} tarball", file=sys.stderr) 181 return ret 182 183 Path(archive).unlink() 184 185 if tag >= "v23" and tag < "v28.2" and args.host == "arm64-apple-darwin": 186 # Starting with v23 there are arm64 binaries for ARM (e.g. M1, M2) mac. 187 # Until v28.2 they had to be signed to run. 188 binary_path = f'{os.getcwd()}/{tag}/bin/' 189 190 for arm_binary in os.listdir(binary_path): 191 # Is it already signed? 192 ret = subprocess.run( 193 ['codesign', '-v', binary_path + arm_binary], 194 stderr=subprocess.DEVNULL, # Suppress expected stderr output 195 ).returncode 196 if ret == 1: 197 # Have to self-sign the binary 198 ret = subprocess.run( 199 ['codesign', '-s', '-', binary_path + arm_binary] 200 ).returncode 201 if ret != 0: 202 print(f"Failed to self-sign {tag} {arm_binary} arm64 binary", file=sys.stderr) 203 return 1 204 205 # Confirm success 206 ret = subprocess.run( 207 ['codesign', '-v', binary_path + arm_binary] 208 ).returncode 209 if ret != 0: 210 print(f"Failed to verify the self-signed {tag} {arm_binary} arm64 binary", file=sys.stderr) 211 return 1 212 213 return 0 214 215 216 def set_host(args) -> int: 217 if platform.system().lower() == 'windows': 218 if platform.machine() != 'AMD64': 219 print('Only 64bit Windows supported', file=sys.stderr) 220 return 1 221 args.host = 'win64' 222 return 0 223 host = os.environ.get('HOST', subprocess.check_output( 224 './depends/config.guess').decode()) 225 platforms = { 226 'aarch64-*-linux*': 'aarch64-linux-gnu', 227 'powerpc64le-*-linux-*': 'powerpc64le-linux-gnu', 228 'riscv64-*-linux*': 'riscv64-linux-gnu', 229 'x86_64-*-linux*': 'x86_64-linux-gnu', 230 'x86_64-apple-darwin*': 'x86_64-apple-darwin', 231 'aarch64-apple-darwin*': 'arm64-apple-darwin', 232 } 233 args.host = '' 234 for pattern, target in platforms.items(): 235 if fnmatch(host, pattern): 236 args.host = target 237 if not args.host: 238 print('Not sure which binary to download for {}'.format(host), file=sys.stderr) 239 return 1 240 return 0 241 242 243 def main(args) -> int: 244 Path(args.target_dir).mkdir(exist_ok=True, parents=True) 245 print("Releases directory: {}".format(args.target_dir)) 246 ret = set_host(args) 247 if ret: 248 return ret 249 with pushd(args.target_dir): 250 for tag in args.tags: 251 ret = download_binary(tag, args) 252 if ret: 253 return ret 254 return 0 255 256 257 if __name__ == '__main__': 258 parser = argparse.ArgumentParser( 259 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 260 epilog=''' 261 HOST can be set to any of the `host-platform-triplet`s from 262 depends/README.md for which a release exists. 263 ''', 264 ) 265 parser.add_argument('-r', '--remove-dir', action='store_true', 266 help='remove existing directory.') 267 parser.add_argument('-t', '--target-dir', action='store', 268 help='target directory.', default='releases') 269 all_tags = sorted([*set([v['tag'] for v in SHA256_SUMS.values()])]) 270 parser.add_argument('tags', nargs='*', default=all_tags, 271 help='release tags. e.g.: v0.18.1 v0.20.0rc2 ' 272 '(if not specified, the full list needed for ' 273 'backwards compatibility tests will be used)' 274 ) 275 args = parser.parse_args() 276 sys.exit(main(args))