get_previous_releases.py
1 #!/usr/bin/env python3 2 # 3 # Copyright (c) 2018-present The Bitcoin Core developers 4 # Distributed under the MIT software license, see the accompanying 5 # file COPYING or http://www.opensource.org/licenses/mit-license.php. 6 # 7 # Download or build previous releases. 8 # Needs curl and tar to download a release, or the build dependencies when 9 # building a release. 10 11 import argparse 12 import contextlib 13 from fnmatch import fnmatch 14 import os 15 from pathlib import Path 16 import re 17 import shutil 18 import subprocess 19 import sys 20 import hashlib 21 22 SHA256_SUMS = { 23 "0e2819135366f150d9906e294b61dff58fd1996ebd26c2f8e979d6c0b7a79580": {"tag": "v0.14.3", "tarball": "bitcoin-0.14.3-aarch64-linux-gnu.tar.gz"}, 24 "d86fc90824a85c38b25c8488115178d5785dbc975f5ff674f9f5716bc8ad6e65": {"tag": "v0.14.3", "tarball": "bitcoin-0.14.3-arm-linux-gnueabihf.tar.gz"}, 25 "1b0a7408c050e3d09a8be8e21e183ef7ee570385dc41216698cc3ab392a484e7": {"tag": "v0.14.3", "tarball": "bitcoin-0.14.3-osx64.tar.gz"}, 26 "706e0472dbc933ed2757650d54cbcd780fd3829ebf8f609b32780c7eedebdbc9": {"tag": "v0.14.3", "tarball": "bitcoin-0.14.3-x86_64-linux-gnu.tar.gz"}, 27 # 28 "d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": {"tag": "v0.15.2", "tarball": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz"}, 29 "54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": {"tag": "v0.15.2", "tarball": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz"}, 30 "87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": {"tag": "v0.15.2", "tarball": "bitcoin-0.15.2-osx64.tar.gz"}, 31 "566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": {"tag": "v0.15.2", "tarball": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz"}, 32 # 33 "0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": {"tag": "v0.16.3", "tarball": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz"}, 34 "fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": {"tag": "v0.16.3", "tarball": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz"}, 35 "78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": {"tag": "v0.16.3", "tarball": "bitcoin-0.16.3-osx64.tar.gz"}, 36 "5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": {"tag": "v0.16.3", "tarball": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz"}, 37 # 38 "5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": {"tag": "v0.17.2", "tarball": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz"}, 39 "d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": {"tag": "v0.17.2", "tarball": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz"}, 40 "a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": {"tag": "v0.17.2", "tarball": "bitcoin-0.17.2-osx64.tar.gz"}, 41 "943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": {"tag": "v0.17.2", "tarball": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz"}, 42 # 43 "88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": {"tag": "v0.18.1", "tarball": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz"}, 44 "cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": {"tag": "v0.18.1", "tarball": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz"}, 45 "b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": {"tag": "v0.18.1", "tarball": "bitcoin-0.18.1-osx64.tar.gz"}, 46 "425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": {"tag": "v0.18.1", "tarball": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz"}, 47 "600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": {"tag": "v0.18.1", "tarball": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz"}, 48 # 49 "3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": {"tag": "v0.19.1", "tarball": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz"}, 50 "657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": {"tag": "v0.19.1", "tarball": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz"}, 51 "1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": {"tag": "v0.19.1", "tarball": "bitcoin-0.19.1-osx64.tar.gz"}, 52 "aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": {"tag": "v0.19.1", "tarball": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz"}, 53 "5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": {"tag": "v0.19.1", "tarball": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz"}, 54 # 55 "60c93e3462c303eb080be7cf623f1a7684b37fd47a018ad3848bc23e13c84e1c": {"tag": "v0.20.1", "tarball": "bitcoin-0.20.1-aarch64-linux-gnu.tar.gz"}, 56 "55b577e0fb306fb429d4be6c9316607753e8543e5946b542d75d876a2f08654c": {"tag": "v0.20.1", "tarball": "bitcoin-0.20.1-arm-linux-gnueabihf.tar.gz"}, 57 "b9024dde373ea7dad707363e07ec7e265383204127539ae0c234bff3a61da0d1": {"tag": "v0.20.1", "tarball": "bitcoin-0.20.1-osx64.tar.gz"}, 58 "fa71cb52ee5e0459cbf5248cdec72df27995840c796f58b304607a1ed4c165af": {"tag": "v0.20.1", "tarball": "bitcoin-0.20.1-riscv64-linux-gnu.tar.gz"}, 59 "376194f06596ecfa40331167c39bc70c355f960280bd2a645fdbf18f66527397": {"tag": "v0.20.1", "tarball": "bitcoin-0.20.1-x86_64-linux-gnu.tar.gz"}, 60 61 "43416854330914992bbba2d0e9adf2a6fff4130be9af8ae2ef1186e743d9a3fe": {"tag": "v0.21.0", "tarball": "bitcoin-0.21.0-aarch64-linux-gnu.tar.gz"}, 62 "f028af308eda45a3c4c90f9332f96b075bf21e3495c945ebce48597151808176": {"tag": "v0.21.0", "tarball": "bitcoin-0.21.0-arm-linux-gnueabihf.tar.gz"}, 63 "695fb624fa6423f5da4f443b60763dd1d77488bfe5ef63760904a7b54e91298d": {"tag": "v0.21.0", "tarball": "bitcoin-0.21.0-osx64.tar.gz"}, 64 "f8b2adfeae021a672effbc7bd40d5c48d6b94e53b2dd660f787340bf1a52e4e9": {"tag": "v0.21.0", "tarball": "bitcoin-0.21.0-riscv64-linux-gnu.tar.gz"}, 65 "da7766775e3f9c98d7a9145429f2be8297c2672fe5b118fd3dc2411fb48e0032": {"tag": "v0.21.0", "tarball": "bitcoin-0.21.0-x86_64-linux-gnu.tar.gz"}, 66 67 "ac718fed08570a81b3587587872ad85a25173afa5f9fbbd0c03ba4d1714cfa3e": {"tag": "v22.0", "tarball": "bitcoin-22.0-aarch64-linux-gnu.tar.gz"}, 68 "b8713c6c5f03f5258b54e9f436e2ed6d85449aa24c2c9972f91963d413e86311": {"tag": "v22.0", "tarball": "bitcoin-22.0-arm-linux-gnueabihf.tar.gz"}, 69 "2744d199c3343b2d94faffdfb2c94d75a630ba27301a70e47b0ad30a7e0155e9": {"tag": "v22.0", "tarball": "bitcoin-22.0-osx64.tar.gz"}, 70 "2cca5f99007d060aca9d8c7cbd035dfe2f040dd8200b210ce32cdf858479f70d": {"tag": "v22.0", "tarball": "bitcoin-22.0-powerpc64-linux-gnu.tar.gz"}, 71 "91b1e012975c5a363b5b5fcc81b5b7495e86ff703ec8262d4b9afcfec633c30d": {"tag": "v22.0", "tarball": "bitcoin-22.0-powerpc64le-linux-gnu.tar.gz"}, 72 "9cc3a62c469fe57e11485fdd32c916f10ce7a2899299855a2e479256ff49ff3c": {"tag": "v22.0", "tarball": "bitcoin-22.0-riscv64-linux-gnu.tar.gz"}, 73 "59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16": {"tag": "v22.0", "tarball": "bitcoin-22.0-x86_64-linux-gnu.tar.gz"}, 74 75 "06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb": {"tag": "v23.0", "tarball": "bitcoin-23.0-aarch64-linux-gnu.tar.gz"}, 76 "952c574366aff76f6d6ad1c9ee45a361d64fa04155e973e926dfe7e26f9703a3": {"tag": "v23.0", "tarball": "bitcoin-23.0-arm-linux-gnueabihf.tar.gz"}, 77 "7c8bc63731aa872b7b334a8a7d96e33536ad77d49029bad179b09dca32cd77ac": {"tag": "v23.0", "tarball": "bitcoin-23.0-arm64-apple-darwin.tar.gz"}, 78 "2caa5898399e415f61d9af80a366a3008e5856efa15aaff74b88acf429674c99": {"tag": "v23.0", "tarball": "bitcoin-23.0-powerpc64-linux-gnu.tar.gz"}, 79 "217dd0469d0f4962d22818c368358575f6a0abcba8804807bb75325eb2f28b19": {"tag": "v23.0", "tarball": "bitcoin-23.0-powerpc64le-linux-gnu.tar.gz"}, 80 "078f96b1e92895009c798ab827fb3fde5f6719eee886bd0c0e93acab18ea4865": {"tag": "v23.0", "tarball": "bitcoin-23.0-riscv64-linux-gnu.tar.gz"}, 81 "c816780583009a9dad426dc0c183c89be9da98906e1e2c7ebae91041c1aaaaf3": {"tag": "v23.0", "tarball": "bitcoin-23.0-x86_64-apple-darwin.tar.gz"}, 82 "2cca490c1f2842884a3c5b0606f179f9f937177da4eadd628e3f7fd7e25d26d0": {"tag": "v23.0", "tarball": "bitcoin-23.0-x86_64-linux-gnu.tar.gz"}, 83 84 "0b48b9e69b30037b41a1e6b78fb7cbcc48c7ad627908c99686e81f3802454609": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-aarch64-linux-gnu.tar.gz"}, 85 "37d7660f0277301744e96426bbb001d2206b8d4505385dfdeedf50c09aaaef60": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-arm-linux-gnueabihf.tar.gz"}, 86 "90ed59e86bfda1256f4b4cad8cc1dd77ee0efec2492bcb5af61402709288b62c": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-arm64-apple-darwin.tar.gz"}, 87 "7590645e8676f8b5fda62dc20174474c4ac8fd0defc83a19ed908ebf2e94dc11": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-powerpc64-linux-gnu.tar.gz"}, 88 "79e89a101f23ff87816675b98769cd1ee91059f95c5277f38f48f21a9f7f8509": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-powerpc64le-linux-gnu.tar.gz"}, 89 "6b163cef7de4beb07b8cb3347095e0d76a584019b1891135cd1268a1f05b9d88": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-riscv64-linux-gnu.tar.gz"}, 90 "e2f751512f3c0f00eb68ba946d9c829e6cf99422a61e8f5e0a7c109c318674d0": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-x86_64-apple-darwin.tar.gz"}, 91 "49df6e444515d457ea0b885d66f521f2a26ca92ccf73d5296082e633544253bf": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-x86_64-linux-gnu.tar.gz"}, 92 93 "3a7bdd959a0b426624f63f394f25e5b7769a5a2f96f8126dcc2ea53f3fa5212b": {"tag": "v25.0", "tarball": "bitcoin-25.0-aarch64-linux-gnu.tar.gz"}, 94 "e537c8630b05e63242d979c3004f851fd73c2a10b5b4fdbb161788427c7b3c0f": {"tag": "v25.0", "tarball": "bitcoin-25.0-arm-linux-gnueabihf.tar.gz"}, 95 "3b35075d6c1209743611c705a13575be2668bc069bc6301ce78a2e1e53ebe7cc": {"tag": "v25.0", "tarball": "bitcoin-25.0-arm64-apple-darwin.tar.gz"}, 96 "0c8e135a6fd297270d3b65196042d761453493a022b5ff7fb847fc911e938214": {"tag": "v25.0", "tarball": "bitcoin-25.0-powerpc64-linux-gnu.tar.gz"}, 97 "fa8af160782f5adfcea570f72b947073c1663b3e9c3cd0f82b216b609fe47573": {"tag": "v25.0", "tarball": "bitcoin-25.0-powerpc64le-linux-gnu.tar.gz"}, 98 "fe6e347a66043946920c72c9c4afca301968101e6b82fb90a63d7885ebcceb32": {"tag": "v25.0", "tarball": "bitcoin-25.0-riscv64-linux-gnu.tar.gz"}, 99 "5708fc639cdfc27347cccfd50db9b73b53647b36fb5f3a4a93537cbe8828c27f": {"tag": "v25.0", "tarball": "bitcoin-25.0-x86_64-apple-darwin.tar.gz"}, 100 "33930d432593e49d58a9bff4c30078823e9af5d98594d2935862788ce8a20aec": {"tag": "v25.0", "tarball": "bitcoin-25.0-x86_64-linux-gnu.tar.gz"}, 101 } 102 103 104 @contextlib.contextmanager 105 def pushd(new_dir) -> None: 106 previous_dir = os.getcwd() 107 os.chdir(new_dir) 108 try: 109 yield 110 finally: 111 os.chdir(previous_dir) 112 113 114 def download_binary(tag, args) -> int: 115 if Path(tag).is_dir(): 116 if not args.remove_dir: 117 print('Using cached {}'.format(tag)) 118 return 0 119 shutil.rmtree(tag) 120 Path(tag).mkdir() 121 bin_path = 'bin/bitcoin-core-{}'.format(tag[1:]) 122 match = re.compile('v(.*)(rc[0-9]+)$').search(tag) 123 if match: 124 bin_path = 'bin/bitcoin-core-{}/test.{}'.format( 125 match.group(1), match.group(2)) 126 platform = args.platform 127 if tag < "v23" and platform in ["x86_64-apple-darwin", "arm64-apple-darwin"]: 128 platform = "osx64" 129 tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( 130 tag=tag[1:], platform=platform) 131 tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format( 132 bin_path=bin_path, tarball=tarball) 133 134 print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl)) 135 136 header, status = subprocess.Popen( 137 ['curl', '--head', tarballUrl], stdout=subprocess.PIPE).communicate() 138 if re.search("404 Not Found", header.decode("utf-8")): 139 print("Binary tag was not found") 140 return 1 141 142 curlCmds = [ 143 ['curl', '--remote-name', tarballUrl] 144 ] 145 146 for cmd in curlCmds: 147 ret = subprocess.run(cmd).returncode 148 if ret: 149 return ret 150 151 hasher = hashlib.sha256() 152 with open(tarball, "rb") as afile: 153 hasher.update(afile.read()) 154 tarballHash = hasher.hexdigest() 155 156 if tarballHash not in SHA256_SUMS or SHA256_SUMS[tarballHash]['tarball'] != tarball: 157 if tarball in [v['tarball'] for v in SHA256_SUMS.values()]: 158 print("Checksum did not match") 159 return 1 160 161 print("Checksum for given version doesn't exist") 162 return 1 163 print("Checksum matched") 164 165 # Extract tarball 166 ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag, 167 '--strip-components=1', 168 'bitcoin-{tag}'.format(tag=tag[1:])]).returncode 169 if ret != 0: 170 print(f"Failed to extract the {tag} tarball") 171 return ret 172 173 Path(tarball).unlink() 174 175 if tag >= "v23" and platform == "arm64-apple-darwin": 176 # Starting with v23 there are arm64 binaries for ARM (e.g. M1, M2) macs, but they have to be signed to run 177 binary_path = f'{os.getcwd()}/{tag}/bin/' 178 179 for arm_binary in os.listdir(binary_path): 180 # Is it already signed? 181 ret = subprocess.run( 182 ['codesign', '-v', binary_path + arm_binary], 183 stderr=subprocess.DEVNULL, # Suppress expected stderr output 184 ).returncode 185 if ret == 1: 186 # Have to self-sign the binary 187 ret = subprocess.run( 188 ['codesign', '-s', '-', binary_path + arm_binary] 189 ).returncode 190 if ret != 0: 191 print(f"Failed to self-sign {tag} {arm_binary} arm64 binary") 192 return 1 193 194 # Confirm success 195 ret = subprocess.run( 196 ['codesign', '-v', binary_path + arm_binary] 197 ).returncode 198 if ret != 0: 199 print(f"Failed to verify the self-signed {tag} {arm_binary} arm64 binary") 200 return 1 201 202 return 0 203 204 205 def build_release(tag, args) -> int: 206 githubUrl = "https://github.com/bitcoin/bitcoin" 207 if args.remove_dir: 208 if Path(tag).is_dir(): 209 shutil.rmtree(tag) 210 if not Path(tag).is_dir(): 211 # fetch new tags 212 subprocess.run( 213 ["git", "fetch", githubUrl, "--tags"]) 214 output = subprocess.check_output(['git', 'tag', '-l', tag]) 215 if not output: 216 print('Tag {} not found'.format(tag)) 217 return 1 218 ret = subprocess.run([ 219 'git', 'clone', f'--branch={tag}', '--depth=1', githubUrl, tag 220 ]).returncode 221 if ret: 222 return ret 223 with pushd(tag): 224 host = args.host 225 if args.depends: 226 with pushd('depends'): 227 ret = subprocess.run(['make', 'NO_QT=1']).returncode 228 if ret: 229 return ret 230 host = os.environ.get( 231 'HOST', subprocess.check_output(['./config.guess'])) 232 config_flags = '--prefix={pwd}/depends/{host} '.format( 233 pwd=os.getcwd(), 234 host=host) + args.config_flags 235 cmds = [ 236 './autogen.sh', 237 './configure {}'.format(config_flags), 238 'make', 239 ] 240 for cmd in cmds: 241 ret = subprocess.run(cmd.split()).returncode 242 if ret: 243 return ret 244 # Move binaries, so they're in the same place as in the 245 # release download 246 Path('bin').mkdir(exist_ok=True) 247 files = ['bitcoind', 'bitcoin-cli', 'bitcoin-tx'] 248 for f in files: 249 Path('src/'+f).rename('bin/'+f) 250 return 0 251 252 253 def check_host(args) -> int: 254 args.host = os.environ.get('HOST', subprocess.check_output( 255 './depends/config.guess').decode()) 256 if args.download_binary: 257 platforms = { 258 'aarch64-*-linux*': 'aarch64-linux-gnu', 259 'powerpc64le-*-linux-*': 'powerpc64le-linux-gnu', 260 'riscv64-*-linux*': 'riscv64-linux-gnu', 261 'x86_64-*-linux*': 'x86_64-linux-gnu', 262 'x86_64-apple-darwin*': 'x86_64-apple-darwin', 263 'aarch64-apple-darwin*': 'arm64-apple-darwin', 264 } 265 args.platform = '' 266 for pattern, target in platforms.items(): 267 if fnmatch(args.host, pattern): 268 args.platform = target 269 if not args.platform: 270 print('Not sure which binary to download for {}'.format(args.host)) 271 return 1 272 return 0 273 274 275 def main(args) -> int: 276 Path(args.target_dir).mkdir(exist_ok=True, parents=True) 277 print("Releases directory: {}".format(args.target_dir)) 278 ret = check_host(args) 279 if ret: 280 return ret 281 if args.download_binary: 282 with pushd(args.target_dir): 283 for tag in args.tags: 284 ret = download_binary(tag, args) 285 if ret: 286 return ret 287 return 0 288 args.config_flags = os.environ.get('CONFIG_FLAGS', '') 289 args.config_flags += ' --without-gui --disable-tests --disable-bench' 290 with pushd(args.target_dir): 291 for tag in args.tags: 292 ret = build_release(tag, args) 293 if ret: 294 return ret 295 return 0 296 297 298 if __name__ == '__main__': 299 parser = argparse.ArgumentParser( 300 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 301 parser.add_argument('-r', '--remove-dir', action='store_true', 302 help='remove existing directory.') 303 parser.add_argument('-d', '--depends', action='store_true', 304 help='use depends.') 305 parser.add_argument('-b', '--download-binary', action='store_true', 306 help='download release binary.') 307 parser.add_argument('-t', '--target-dir', action='store', 308 help='target directory.', default='releases') 309 all_tags = sorted([*set([v['tag'] for v in SHA256_SUMS.values()])]) 310 parser.add_argument('tags', nargs='*', default=all_tags, 311 help='release tags. e.g.: v0.18.1 v0.20.0rc2 ' 312 '(if not specified, the full list needed for ' 313 'backwards compatibility tests will be used)' 314 ) 315 args = parser.parse_args() 316 sys.exit(main(args))