1  import os
  2  import random
  3  
  4  from flask_babel import Babel
  5  from flask_debugtoolbar import DebugToolbarExtension
  6  from flask_static_digest import FlaskStaticDigest
  7  from sqlalchemy import Column, Integer, ForeignKey, inspect, create_engine, Text
  8  from sqlalchemy.orm import declarative_base, relationship
  9  from sqlalchemy.ext.declarative import DeferredReflection
 10  from elasticsearch import Elasticsearch
 11  from flask_mail import Mail
 12  from config.settings import ELASTICSEARCH_HOST, ELASTICSEARCHAUX_HOST, ELASTICSEARCH_HOST_PREFERRED, ELASTICSEARCHAUX_HOST_PREFERRED
 13  
 14  debug_toolbar = DebugToolbarExtension()
 15  flask_static_digest = FlaskStaticDigest()
 16  Base = declarative_base()
 17  babel = Babel()
 18  mail = Mail()
 19  
 20  class FallbackNodeSelector: # Selects only the first live node
 21      def __init__(self, node_configs):
 22          self.node_configs = node_configs
 23      def select(self, nodes):
 24          node_configs = list(self.node_configs)
 25          reverse = (random.randint(0, 100) < 10)
 26          if reverse:
 27              node_configs.reverse() # Occasionally pick the fallback to check it.
 28          for node_config in node_configs:
 29              for node in nodes:
 30                  if node.config == node_config:
 31                      if node_config != self.node_configs[0]:
 32                          print(f"FallbackNodeSelector warning: using fallback node! {reverse=} {node_config=}")
 33                      return node
 34          raise Exception("No node_config found!")
 35  
 36  if len(ELASTICSEARCH_HOST_PREFERRED) > 0:
 37      es = Elasticsearch(hosts=[ELASTICSEARCH_HOST_PREFERRED,ELASTICSEARCH_HOST], node_selector_class=FallbackNodeSelector, max_retries=1, retry_on_timeout=False, http_compress=True, randomize_hosts=False)
 38  else:
 39      es = Elasticsearch(hosts=[ELASTICSEARCH_HOST], max_retries=1, retry_on_timeout=False, http_compress=False, randomize_hosts=False)
 40  if len(ELASTICSEARCHAUX_HOST_PREFERRED) > 0:
 41      # Let's not fall back here, because ELASTICSEARCHAUX_HOST is just so slow..
 42      es_aux = Elasticsearch(hosts=[ELASTICSEARCHAUX_HOST_PREFERRED], max_retries=1, retry_on_timeout=False, http_compress=True, randomize_hosts=False)
 43  else:
 44      es_aux = Elasticsearch(hosts=[ELASTICSEARCHAUX_HOST], max_retries=1, retry_on_timeout=False, http_compress=False, randomize_hosts=False)
 45  
 46  mariadb_user = os.getenv("MARIADB_USER", "allthethings")
 47  mariadb_password = os.getenv("MARIADB_PASSWORD", "password")
 48  mariadb_host = os.getenv("MARIADB_HOST", "mariadb")
 49  mariadb_port = os.getenv("MARIADB_PORT", "3306")
 50  mariadb_db = os.getenv("MARIADB_DATABASE", mariadb_user)
 51  mariadb_url = f"mysql+pymysql://{mariadb_user}:{mariadb_password}@{mariadb_host}:{mariadb_port}/{mariadb_db}?read_timeout=120&write_timeout=120"
 52  mariadb_url_no_timeout = f"mysql+pymysql://root:{mariadb_password}@{mariadb_host}:{mariadb_port}/{mariadb_db}"
 53  if os.getenv("DATA_IMPORTS_MODE", "") == "1":
 54      mariadb_url = mariadb_url_no_timeout
 55  engine = create_engine(mariadb_url, future=True, isolation_level="AUTOCOMMIT", pool_size=5, max_overflow=0, pool_recycle=300, pool_pre_ping=True)
 56  
 57  mariapersist_user = os.getenv("MARIAPERSIST_USER", "allthethings")
 58  mariapersist_password = os.getenv("MARIAPERSIST_PASSWORD", "password")
 59  mariapersist_host = os.getenv("MARIAPERSIST_HOST", "mariapersist")
 60  mariapersist_port = os.getenv("MARIAPERSIST_PORT", "3333")
 61  mariapersist_db = os.getenv("MARIAPERSIST_DATABASE", mariapersist_user)
 62  mariapersist_url = f"mysql+pymysql://{mariapersist_user}:{mariapersist_password}@{mariapersist_host}:{mariapersist_port}/{mariapersist_db}?read_timeout=120&write_timeout=120"
 63  mariapersist_engine = create_engine(mariapersist_url, future=True, isolation_level="AUTOCOMMIT", pool_size=5, max_overflow=0, pool_recycle=300, pool_pre_ping=True)
 64  
 65  class Reflected(DeferredReflection, Base):
 66      __abstract__ = True
 67      def to_dict(self):
 68          unloaded = inspect(self).unloaded
 69          return dict((col.name, getattr(self, col.name)) for col in self.__table__.columns if col.name not in unloaded)
 70  
 71  class ReflectedMariapersist(DeferredReflection, Base):
 72      __abstract__ = True
 73      def to_dict(self):
 74          unloaded = db.inspect(self).unloaded
 75          return dict((col.name, getattr(self, col.name)) for col in self.__table__.columns if col.name not in unloaded)
 76  
 77  class ZlibBook(Reflected):
 78      __tablename__ = "zlib_book"
 79      isbns = relationship("ZlibIsbn", lazy="selectin")
 80  class ZlibIsbn(Reflected):
 81      __tablename__ = "zlib_isbn"
 82      zlibrary_id = Column(Integer, ForeignKey("zlib_book.zlibrary_id"))
 83  
 84  class IsbndbIsbns(Reflected):
 85      __tablename__ = "isbndb_isbns"
 86  
 87  class LibgenliFiles(Reflected):
 88      __tablename__ = "libgenli_files"
 89      add_descrs = relationship("LibgenliFilesAddDescr", lazy="selectin")
 90      editions = relationship("LibgenliEditions", lazy="selectin", secondary="libgenli_editions_to_files")
 91  class LibgenliFilesAddDescr(Reflected):
 92      __tablename__ = "libgenli_files_add_descr"
 93      f_id = Column(Integer, ForeignKey("libgenli_files.f_id"))
 94  class LibgenliEditionsToFiles(Reflected):
 95      __tablename__ = "libgenli_editions_to_files"
 96      f_id = Column(Integer, ForeignKey("libgenli_files.f_id"))
 97      e_id = Column(Integer, ForeignKey("libgenli_editions.e_id"))
 98  class LibgenliEditions(Reflected):
 99      __tablename__ = "libgenli_editions"
100      issue_s_id = Column(Integer, ForeignKey("libgenli_series.s_id"))
101      series = relationship("LibgenliSeries", lazy="joined")
102      add_descrs = relationship("LibgenliEditionsAddDescr", lazy="selectin")
103  class LibgenliEditionsAddDescr(Reflected):
104      __tablename__ = "libgenli_editions_add_descr"
105      e_id = Column(Integer, ForeignKey("libgenli_editions.e_id"))
106      publisher = relationship("LibgenliPublishers", lazy="joined", primaryjoin="(remote(LibgenliEditionsAddDescr.value) == foreign(LibgenliPublishers.p_id)) & (LibgenliEditionsAddDescr.key == 308)")
107  class LibgenliPublishers(Reflected):
108      __tablename__ = "libgenli_publishers"
109  class LibgenliSeries(Reflected):
110      __tablename__ = "libgenli_series"
111      issn_add_descrs = relationship("LibgenliSeriesAddDescr", lazy="joined", primaryjoin="(LibgenliSeries.s_id == LibgenliSeriesAddDescr.s_id) & (LibgenliSeriesAddDescr.key == 501)")
112  class LibgenliSeriesAddDescr(Reflected):
113      __tablename__ = "libgenli_series_add_descr"
114      s_id = Column(Integer, ForeignKey("libgenli_series.s_id"))
115  class LibgenliElemDescr(Reflected):
116      __tablename__ = "libgenli_elem_descr"
117  
118  class LibgenrsDescription(Reflected):
119      __tablename__ = "libgenrs_description"
120  class LibgenrsHashes(Reflected):
121      __tablename__ = "libgenrs_hashes"
122  class LibgenrsTopics(Reflected):
123      __tablename__ = "libgenrs_topics"
124  class LibgenrsUpdated(Reflected):
125      __tablename__ = "libgenrs_updated"
126  
127  class LibgenrsFiction(Reflected):
128      __tablename__ = "libgenrs_fiction"
129  class LibgenrsFictionDescription(Reflected):
130      __tablename__ = "libgenrs_fiction_description"
131  class LibgenrsFictionHashes(Reflected):
132      __tablename__ = "libgenrs_fiction_hashes"
133  
134  class OlBase(Reflected):
135      __tablename__ = "ol_base"
136  
137  class AaIa202306Metadata(Reflected):
138      __tablename__ = "aa_ia_2023_06_metadata"
139  class AaIa202306Files(Reflected):
140      __tablename__ = "aa_ia_2023_06_files"
141  class Ia2Records(Reflected):
142      __tablename__ = "annas_archive_meta__aacid__ia2_records"
143  class Ia2AcsmpdfFiles(Reflected):
144      __tablename__ = "annas_archive_meta__aacid__ia2_acsmpdf_files"
145  
146  
147  class MariapersistDownloadsTotalByMd5(ReflectedMariapersist):
148      __tablename__ = "mariapersist_downloads_total_by_md5"
149  class MariapersistAccounts(ReflectedMariapersist):
150      __tablename__ = "mariapersist_accounts"
151  class MariapersistDownloads(ReflectedMariapersist):
152      __tablename__ = "mariapersist_downloads"
153  class MariapersistDownloadsHourlyByMd5(ReflectedMariapersist):
154      __tablename__ = "mariapersist_downloads_hourly_by_md5"
155  class MariapersistDownloadsHourly(ReflectedMariapersist):
156      __tablename__ = "mariapersist_downloads_hourly"
157  class MariapersistMd5Report(ReflectedMariapersist):
158      __tablename__ = "mariapersist_md5_report"
159  class MariapersistComments(ReflectedMariapersist):
160      __tablename__ = "mariapersist_comments"
161  class MariapersistReactions(ReflectedMariapersist):
162      __tablename__ = "mariapersist_reactions"
163  class MariapersistLists(ReflectedMariapersist):
164      __tablename__ = "mariapersist_lists"
165  class MariapersistListEntries(ReflectedMariapersist):
166      __tablename__ = "mariapersist_list_entries"
167  class MariapersistDonations(ReflectedMariapersist):
168      __tablename__ = "mariapersist_donations"
169  class MariapersistCopyrightClaims(ReflectedMariapersist):
170      __tablename__ = "mariapersist_copyright_claims"
171  class MariapersistFastDownloadAccess(ReflectedMariapersist):
172      __tablename__ = "mariapersist_fast_download_access"
173  class MariapersistSmallFiles(ReflectedMariapersist):
174      __tablename__ = "mariapersist_small_files"
175  # class MariapersistSearches(ReflectedMariapersist):
176  #     __tablename__ = "mariapersist_searches"
177  
178