1  #!/bin/bash
 2  
 3  set -Eeuxo pipefail
 4  # https://stackoverflow.com/a/3355423
 5  cd "$(dirname "$0")"
 6  
 7  # Run this script by running: docker exec -it aa-data-import--web /scripts/load_libgenrs.sh
 8  # Feel free to comment out steps in order to retry failed parts of this script, when necessary.
 9  # Load scripts are idempotent, and can be rerun without losing too much work.
10  
11  cd /temp-dir
12  
13  rm -f libgen.sql fiction.sql
14  
15  unrar e libgen.rar &
16  job1pid=$!
17  unrar e fiction.rar &
18  job2pid=$!
19  wait $job1pid
20  wait $job2pid
21  
22  pv libgen.sql  | PYTHONIOENCODING=UTF8:ignore python3 /scripts/helpers/sanitize_unicode.py | mariadb -h aa-data-import--mariadb --default-character-set=utf8mb4 -u root -ppassword allthethings &
23  job1pid=$!
24  pv fiction.sql | PYTHONIOENCODING=UTF8:ignore python3 /scripts/helpers/sanitize_unicode.py | mariadb -h aa-data-import--mariadb --default-character-set=utf8mb4 -u root -ppassword allthethings &
25  job2pid=$!
26  wait $job1pid
27  wait $job2pid
28  
29  mariadb -h aa-data-import--mariadb -u root -ppassword allthethings --show-warnings -vv < /scripts/helpers/libgenrs_final.sql
30  
31  rm libgen.sql fiction.sql