1  'use strict';
   2  var ext_api = (typeof browser === 'object') ? browser : chrome;
   3  var url_loc = (typeof browser === 'object') ? 'firefox' : 'chrome';
   4  var manifestData = ext_api.runtime.getManifest();
   5  var ext_name = manifestData.name;
   6  var ext_version = manifestData.version;
   7  var navigator_ua = navigator.userAgent;
   8  var navigator_ua_mobile = navigator_ua.toLowerCase().includes('mobile');
   9  var kiwi_browser = navigator_ua_mobile && (url_loc === 'chrome') && !navigator_ua.toLowerCase().includes('yabrowser');
  10  
  11  if (typeof ext_api.action !== 'object') {
  12    ext_api.action = ext_api.browserAction;
  13  }
  14  
  15  var dompurify_sites = [];
  16  var optin_setcookie = false;
  17  var optin_update = true;
  18  var blocked_referer = false;
  19  
  20  // defaultSites are loaded from sites.js at installation extension
  21  
  22  var restrictions = {
  23    'bloomberg.com': /^((?!\.bloomberg\.com\/news\/terminal\/).)*$/,
  24    'bloombergadria.com': /^((?!\.bloombergadria\.com\/video\/).)*$/,
  25    'dailywire.com': /^((?!\.dailywire\.com\/(episode|show|videos|watch)).)*$/,
  26    'economictimes.com': /\.economictimes\.com($|\/($|(__assets|prime)(\/.+)?|.+\.cms))/,
  27    'elespanol.com': /^((?!\/cronicaglobal\.elespanol\.com\/).)*$/,
  28    'espn.com': /^((?!espn\.com\/watch).)*$/,
  29    'esquire.com': /^((?!\/classic\.esquire\.com\/).)*$/,
  30    'foreignaffairs.com': /^((?!\/reader\.foreignaffairs\.com\/).)*$/,
  31    'hilltimes.com': /^((?!hilltimes\.com\/slideshow\/).)*$/,
  32    'nytimes.com': /^((?!\/timesmachine\.nytimes\.com\/).)*$/,
  33    'science.org': /^((?!\.science\.org\/doi\/).)*$/,
  34    'timesofindia.com': /\.timesofindia\.com($|\/($|toi-plus(\/.+)?|.+\.cms))/,
  35    'quora.com': /^((?!quora\.com\/search\?q=).)*$/,
  36    'seekingalpha.com': /\/seekingalpha\.com($|\/($|(amp\/)?(article|news)\/|samw\/))/,
  37    'statista.com': /^((?!\.statista\.com\/study\/).)*$/,
  38    'techinasia.com': /\.techinasia\.com\/.+/,
  39    'theatlantic.com': /^((?!\/newsletters\.theatlantic\.com\/).)*$/,
  40    'thetimes.co.uk': /^((?!epaper\.thetimes\.co\.uk).)*$/,
  41    'timeshighereducation.com': /\.timeshighereducation\.com\/((features|news|people)\/|.+((\w)+(\-)+){3,}.+|sites\/default\/files\/)/,
  42    'uol.com.br': /^((?!(conta|email)\.uol\.com\.br).)*$/,
  43  }
  44  
  45  for (let domain of au_news_corp_domains)
  46    restrictions[domain] = new RegExp('^((?!todayspaper\\.' + domain.replace(/\./g, '\\.') + '\\/).)*$');
  47  if (typeof browser !== 'object') {
  48    for (let domain of [])
  49      restrictions[domain] = new RegExp('((\\/|\\.)' + domain.replace(/\./g, '\\.') + '\\/$|' + restrictions[domain].toString().replace(/(^\/|\/$)/g, '') + ')');
  50  }
  51  
  52  // Don't remove cookies before/after page load
  53  var allow_cookies = [];
  54  var remove_cookies = [];
  55  // select specific cookie(s) to hold/drop from remove_cookies domains
  56  var remove_cookies_select_hold, remove_cookies_select_drop;
  57  
  58  // Set User-Agent
  59  var use_google_bot, use_bing_bot, use_facebook_bot;
  60  // Set Referer
  61  var use_facebook_referer, use_google_referer, use_twitter_referer;
  62  // Set random IP-address
  63  var random_ip = {};
  64  var use_random_ip = [];
  65  // concat all sites with change of headers (useragent, referer or random ip)
  66  var change_headers;
  67  
  68  // block paywall-scripts
  69  var blockedRegexes = {};
  70  var blockedRegexesDomains = [];
  71  var blockedRegexesGeneral = {};
  72  var blockedJsInline = {};
  73  var blockedJsInlineDomains = [];
  74  
  75  // unhide text on amp-page
  76  var amp_unhide;
  77  // redirect to amp-page
  78  var amp_redirect;
  79  // code for contentScript
  80  var cs_code;
  81  // load text from json (script[type="application/ld+json"])
  82  var ld_json;
  83  // load text from json (script#__NEXT_DATA__)
  84  var ld_json_next;
  85  // load text from Google webcache
  86  var ld_google_webcache;
  87  // add external link to article
  88  var add_ext_link;
  89  
  90  // custom: block javascript
  91  var block_js_custom = [];
  92  var block_js_custom_ext = [];
  93  
  94  function initSetRules() {
  95    allow_cookies = [];
  96    remove_cookies = [];
  97    remove_cookies_select_drop = {};
  98    remove_cookies_select_hold = {};
  99    use_google_bot = [];
 100    use_bing_bot = [];
 101    use_facebook_bot = [];
 102    use_facebook_referer = [];
 103    use_google_referer = [];
 104    use_twitter_referer = [];
 105    random_ip = {};
 106    change_headers = [];
 107    amp_unhide = [];
 108    amp_redirect = {};
 109    cs_code = {};
 110    ld_json = {};
 111    ld_json_next = {};
 112    ld_google_webcache = {};
 113    add_ext_link = {};
 114    block_js_custom = [];
 115    block_js_custom_ext = [];
 116    blockedRegexes = {};
 117    blockedRegexesDomains = [];
 118    blockedRegexesGeneral = {};
 119    blockedJsInline = {};
 120    blockedJsInlineDomains = [];
 121    init_custom_flex_domains();
 122  }
 123  
 124  const userAgentDesktopG = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
 125  const userAgentMobileG = "Chrome/80.0.3987.92 Mobile Safari/537.36 (compatible ; Googlebot/2.1 ; +http://www.google.com/bot.html)";
 126  
 127  const userAgentDesktopB = "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)";
 128  const userAgentMobileB = "Chrome/80.0.3987.92 Mobile Safari/537.36 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)";
 129  
 130  const userAgentDesktopF = 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)';
 131  
 132  var enabledSites = [];
 133  var disabledSites = [];
 134  var optionSites = {};
 135  var customSites = {};
 136  var customSites_domains = [];
 137  var updatedSites = {};
 138  var updatedSites_new = [];
 139  var updatedSites_domains_new = [];
 140  var excludedSites = [];
 141  
 142  function setDefaultOptions() {
 143    ext_api.storage.local.set({
 144      sites: filterObject(defaultSites, function (val, key) {
 145        return val.domain && !val.domain.match(/^(###$|#options_(disable|optin)_)/)
 146      },
 147        function (val, key) {
 148        return [key, val.domain]
 149      })
 150    }, function () {
 151      ext_api.runtime.openOptionsPage();
 152    });
 153  }
 154  
 155  function check_sites_updated() {
 156    let sites_updated_json = 'https://gitlab.com/magnolia1234/bypass-paywalls-' + url_loc + '-clean/-/raw/master/sites_updated.json';
 157    fetch(sites_updated_json)
 158    .then(response => {
 159      if (response.ok) {
 160        response.json().then(json => {
 161          expandSiteRules(json, true);
 162          ext_api.storage.local.set({
 163            sites_updated: json
 164          });
 165        })
 166      }
 167    }).catch(function (err) {
 168      false;
 169    });
 170  }
 171  
 172  function clear_sites_updated() {
 173    ext_api.storage.local.set({
 174      sites_updated: {}
 175    });
 176  }
 177  
 178  function prep_regex_str(str, domain = '') {
 179    if (domain)
 180      str = str.replace(/{domain}/g, domain.replace(/\./g, '\\.'));
 181    return str.replace(/^\//, '').replace(/\/\//g, '/').replace(/([^\\])\/$/, "$1")
 182  }
 183  
 184  function addRules(domain, rule) {
 185    if (rule.hasOwnProperty('remove_cookies_select_drop') || rule.hasOwnProperty('remove_cookies_select_hold')) {
 186      rule.allow_cookies = 1;
 187      rule.remove_cookies = 1;
 188    }
 189    if (rule.allow_cookies > 0 && !allow_cookies.includes(domain))
 190      allow_cookies.push(domain);
 191    if (rule.remove_cookies > 0 && !remove_cookies.includes(domain))
 192      remove_cookies.push(domain);
 193    if (rule.hasOwnProperty('remove_cookies_select_drop'))
 194      remove_cookies_select_drop[domain] = rule.remove_cookies_select_drop;
 195    if (rule.hasOwnProperty('remove_cookies_select_hold'))
 196      remove_cookies_select_hold[domain] = rule.remove_cookies_select_hold;
 197    if (rule.hasOwnProperty('block_regex')) {
 198      if (rule.block_regex instanceof RegExp)
 199        blockedRegexes[domain] = rule.block_regex;
 200      else {
 201        try {
 202          blockedRegexes[domain] = new RegExp(prep_regex_str(rule.block_regex, domain));
 203        } catch (e) {
 204          console.log(`regex not valid, error: ${e}`);
 205        }
 206      }
 207    }
 208    if (rule.hasOwnProperty('block_regex_general')) {
 209      if (rule.block_regex_general instanceof RegExp)
 210        blockedRegexesGeneral[domain] = {block_regex: rule.block_regex_general};
 211      else {
 212        try {
 213          blockedRegexesGeneral[domain] = {block_regex: new RegExp(prep_regex_str(rule.block_regex_general, domain))};
 214        } catch (e) {
 215          console.log(`regex not valid, error: ${e}`);
 216        }
 217      }
 218      blockedRegexesGeneral[domain]['excluded_domains'] = rule.excluded_domains ? rule.excluded_domains : [];
 219    }
 220    if (rule.hasOwnProperty('block_js_inline')) {
 221      if (rule.block_js_inline instanceof RegExp)
 222        blockedJsInline[domain] = rule.block_js_inline;
 223      else {
 224        try {
 225          blockedJsInline[domain] = new RegExp(prep_regex_str(rule.block_js_inline, domain));
 226        } catch (e) {
 227          console.log(`regex not valid, error: ${e}`);
 228        }
 229      }
 230    }
 231    if (rule.useragent) {
 232      switch (rule.useragent) {
 233      case 'googlebot':
 234        if (!use_google_bot.includes(domain))
 235          use_google_bot.push(domain);
 236        break;
 237      case 'bingbot':
 238        if (!use_bing_bot.includes(domain))
 239          use_bing_bot.push(domain);
 240        break;
 241      case 'facebookbot':
 242        if (!use_facebook_bot.includes(domain))
 243          use_facebook_bot.push(domain);
 244        break;
 245      }
 246    }
 247    if (rule.referer) {
 248      switch (rule.referer) {
 249      case 'facebook':
 250        if (!use_facebook_referer.includes(domain))
 251          use_facebook_referer.push(domain);
 252        break;
 253      case 'google':
 254        if (!use_google_referer.includes(domain))
 255          use_google_referer.push(domain);
 256        break;
 257      case 'twitter':
 258        if (!use_twitter_referer.includes(domain))
 259          use_twitter_referer.push(domain);
 260        break;
 261      }
 262    }
 263    if (rule.random_ip) {
 264      random_ip[domain] = rule.random_ip;
 265    }
 266    if (rule.amp_unhide > 0 && !amp_unhide.includes(domain))
 267      amp_unhide.push(domain);
 268    if (rule.amp_redirect)
 269      amp_redirect[domain] = rule.amp_redirect.paywall ? rule.amp_redirect : {paywall: rule.amp_redirect};
 270    if (rule.cs_code) {
 271      if (typeof rule.cs_code === 'string') {
 272        try {
 273          rule.cs_code = JSON.parse(rule.cs_code);
 274        } catch (e) {
 275          console.log(`cs_code not valid: ${rule.cs_code} error: ${e}`);
 276        }
 277      }
 278      if (typeof rule.cs_code === 'object')
 279        cs_code[domain] = rule.cs_code;
 280    }
 281    if (rule.ld_json)
 282      ld_json[domain] = rule.ld_json;
 283    if (rule.ld_json_next)
 284      ld_json_next[domain] = rule.ld_json_next;
 285    if (rule.ld_google_webcache)
 286      ld_google_webcache[domain] = rule.ld_google_webcache;
 287    if (rule.ld_json || rule.ld_json_next || rule.ld_google_webcache || rule.cs_dompurify)
 288      if (!dompurify_sites.includes(domain))
 289        dompurify_sites.push(domain);
 290    if (rule.add_ext_link && rule.add_ext_link_type)
 291      add_ext_link[domain] = {css: rule.add_ext_link, type: rule.add_ext_link_type};
 292  
 293    // custom
 294    if (rule.googlebot > 0)
 295      use_google_bot.push(domain); // legacy
 296    if (rule.block_js > 0 || rule.block_javascript > 0)
 297      block_js_custom.push(domain);
 298    if (rule.block_js_ext > 0 || rule.block_javascript_ext > 0)
 299      block_js_custom_ext.push(domain);
 300  }
 301  
 302  function customFlexAddRules(custom_domain, rule) {
 303    addRules(custom_domain, rule);
 304    if (blockedRegexes[custom_domain])
 305      blockedRegexesDomains.push(custom_domain);
 306    if (blockedJsInline[custom_domain]) {
 307      blockedJsInlineDomains.push(custom_domain);
 308      disableJavascriptInline();
 309    }
 310    if (rule.useragent || rule.referer || rule.random_ip)
 311      change_headers.push(custom_domain);
 312    if (rule.random_ip)
 313      use_random_ip.push(custom_domain);
 314    ext_api.tabs.reload({bypassCache: true});
 315  }
 316  
 317  function set_rules(sites, sites_updated, sites_custom) {
 318    initSetRules();
 319    for (let site in sites) {
 320      let site_domain = sites[site].toLowerCase();
 321      let custom = false;
 322      if (!site_domain.match(/^(###$|#options_)/)) {
 323        let rule = {};
 324        let site_default = defaultSites.hasOwnProperty(site) ? site : Object.keys(defaultSites).find(default_key => compareKey(default_key, site));
 325        if (site_default) {
 326          rule = defaultSites[site_default];
 327          let site_updated = Object.keys(sites_updated).find(updated_key => compareKey(updated_key, site));
 328          if (site_updated && !(sites_updated[site_updated].new_site || (sites_updated[site_updated].upd_version && (sites_updated[site_updated].upd_version <= ext_version))))
 329            rule = sites_updated[site_updated];
 330        } else if (sites_updated.hasOwnProperty(site)) { // updated (new) sites
 331          rule = sites_updated[site];
 332        } else if (sites_custom.hasOwnProperty(site)) { // custom (new) sites
 333          rule = sites_custom[site];
 334          custom = true;
 335        } else
 336          continue;
 337        let domains = [site_domain];
 338        let group = false;
 339        if (rule.hasOwnProperty('group')) {
 340          domains = rule.group;
 341          group = true;
 342        }
 343        let rule_default = {};
 344        if (rule.hasOwnProperty('exception')) {
 345          for (let key in rule)
 346            rule_default[key] = rule[key];
 347        }
 348        for (let domain of domains) {
 349          let custom_in_group = false;
 350          if (rule_default.hasOwnProperty('exception')) {
 351            let exception_rule = rule_default.exception.filter(x => domain === x.domain || (typeof x.domain !== 'string' && x.domain.includes(domain)));
 352            if (exception_rule.length > 0)
 353              rule = exception_rule[0];
 354            else
 355              rule = rule_default;
 356          }
 357          // custom domain for default site(group)
 358          if (!custom) {
 359            let isCustomSite = matchDomain(customSites_domains, domain);
 360            let customSite_title = isCustomSite ? Object.keys(customSites).find(key => customSites[key].domain === isCustomSite) : '';
 361            if (customSite_title && !(rule.add_ext_link || ['swarajyamag.com'].includes(isCustomSite))) {
 362              // add default block_regex
 363              let block_regex_default = '';
 364              if (rule.hasOwnProperty('block_regex'))
 365                block_regex_default = rule.block_regex;
 366              rule = {};
 367              for (let key in sites_custom[customSite_title])
 368                rule[key] = sites_custom[customSite_title][key];
 369              if (block_regex_default) {
 370                if (rule.hasOwnProperty('block_regex')) {
 371                  if (block_regex_default instanceof RegExp)
 372                    block_regex_default = block_regex_default.source;
 373                  rule.block_regex = '(' + block_regex_default + '|' + prep_regex_str(rule.block_regex, domain) + ')';
 374                } else
 375                  rule.block_regex = block_regex_default;
 376              }
 377              if (group)
 378                custom_in_group = true;
 379              else
 380                custom = true;
 381            }
 382          }
 383          addRules(domain, rule);
 384        }
 385      }
 386    }
 387    blockedRegexesDomains = Object.keys(blockedRegexes);
 388    blockedJsInlineDomains = Object.keys(blockedJsInline);
 389    disableJavascriptInline();
 390    use_random_ip = Object.keys(random_ip);
 391    change_headers = use_google_bot.concat(use_bing_bot, use_facebook_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
 392  }
 393  
 394  // add grouped sites to en/disabledSites (and exclude sites)
 395  function add_grouped_enabled_domains(groups) {
 396    for (let key in groups) {
 397      if (enabledSites.includes(key))
 398        enabledSites = enabledSites.concat(groups[key]);
 399      else
 400        disabledSites = disabledSites.concat(groups[key]);
 401      for (let site of excludedSites) {
 402        if (enabledSites.includes(site)) {
 403          enabledSites.splice(enabledSites.indexOf(site), 1);
 404          disabledSites.push(site);
 405        }
 406      }
 407    }
 408  }
 409  
 410  // Get the enabled sites (from local storage) & set_rules for sites
 411  ext_api.storage.local.get({
 412    sites: {},
 413    sites_default: Object.keys(defaultSites).filter(x => defaultSites[x].domain && !defaultSites[x].domain.match(/^(#options_|###$)/)),
 414    sites_custom: {},
 415    sites_updated: {},
 416    sites_excluded: [],
 417    ext_version_old: '2.3.9.0',
 418    optIn: false,
 419    optInUpdate: true
 420  }, function (items) {
 421    var sites = items.sites;
 422    optionSites = sites;
 423    var sites_default = items.sites_default;
 424    customSites = items.sites_custom;
 425    customSites = filterObject(customSites, function (val, key) {
 426      return !(val.add_ext_link && (!val.add_ext_link_type || val.add_ext_link_type === 'google_search_tool'))
 427    });
 428    customSites_domains = Object.values(customSites).map(x => x.domain);
 429    updatedSites = items.sites_updated;
 430    updatedSites_domains_new = Object.values(updatedSites).filter(x => (x.domain && !defaultSites_domains.includes(x.domain) || x.group)).map(x => x.group ? x.group.filter(y => !defaultSites_domains.includes(y)) : x.domain).flat();
 431    var ext_version_old = items.ext_version_old;
 432    optin_setcookie = items.optIn;
 433    optin_update = items.optInUpdate;
 434    excludedSites = items.sites_excluded;
 435  
 436    enabledSites = Object.values(sites).filter(function (val) {
 437      return (val && val !== '###' && (defaultSites_domains.concat(customSites_domains, updatedSites_domains_new).includes(val)));
 438    }).map(function (val) {
 439      return val.toLowerCase();
 440    });
 441  
 442    // Enable new sites by default (opt-in)
 443    updatedSites_new = Object.keys(updatedSites).filter(x => updatedSites[x].domain && !defaultSites_domains.includes(updatedSites[x].domain) && updatedSites[x].domain !== '###_usa_theathletic');
 444    for (let site_updated_new of updatedSites_new)
 445      defaultSites[site_updated_new] = updatedSites[site_updated_new];
 446    if (ext_version > ext_version_old || updatedSites_new.length > 0) {
 447      if (enabledSites.includes('#options_enable_new_sites')) {
 448        let sites_new = Object.keys(defaultSites).filter(x => defaultSites[x].domain && !defaultSites[x].domain.match(/^(#options_|###$)/) && !sites_default.some(key => compareKey(key, x)));
 449        for (let site_new of sites_new)
 450          sites[site_new] = defaultSites[site_new].domain;
 451        // reset ungrouped sites
 452        let ungrouped_sites = {
 453          'The Athletic': 'theathletic.com'
 454        };
 455        for (let key in ungrouped_sites) {
 456          if (sites[key] && sites[key] !== ungrouped_sites[key])
 457            sites[key] = ungrouped_sites[key];
 458        }
 459        ext_api.storage.local.set({
 460          sites: sites
 461        });
 462      } else {
 463        ext_api.management.getSelf(function (result) {
 464          if ((result.installType === 'development' || (result.installType !== 'development' && !enabledSites.includes('#options_on_update')))) {
 465            let new_groups = ['###_de_mhs', '###_nl_eu_ftm'];
 466            let open_options = new_groups.some(group => !enabledSites.includes(group) && grouped_sites[group].some(domain => enabledSites.includes(domain) && !customSites_domains.includes(domain)));
 467            if (open_options)
 468              ext_api.runtime.openOptionsPage();
 469          }
 470        });
 471      }
 472      sites_default = Object.keys(defaultSites).filter(x => defaultSites[x].domain && !defaultSites[x].domain.match(/^(#options_|###$)/));
 473      ext_api.storage.local.set({
 474        sites_default: sites_default,
 475        ext_version_old: ext_version
 476      });
 477    }
 478  
 479    disabledSites = defaultSites_grouped_domains.concat(customSites_domains).filter(x => !enabledSites.includes(x));
 480    add_grouped_enabled_domains(grouped_sites);
 481    set_rules(sites, updatedSites, customSites);
 482    if (enabledSites.includes('#options_optin_update_rules')) {
 483      check_sites_updated();
 484      sites_custom_ext_json = 'https://gitlab.com/magnolia1234/bypass-paywalls-' + url_loc + '-clean/-/raw/master/custom/sites_custom.json';
 485    } 
 486    check_sites_custom_ext();
 487    if (optin_update)
 488      check_update();
 489  });
 490  
 491  // Listen for changes to options
 492  ext_api.storage.onChanged.addListener(function (changes, namespace) {
 493    if (namespace === 'sync')
 494      return;
 495    for (let key in changes) {
 496      var storageChange = changes[key];
 497      if (key === 'sites') {
 498        var sites = storageChange.newValue;
 499        optionSites = sites;
 500        enabledSites = Object.values(sites).filter(function (val) {
 501          return (val && val !== '###' && (defaultSites_domains.concat(customSites_domains, updatedSites_domains_new).includes(val)));
 502        }).map(function (val) {
 503          return val.toLowerCase();
 504        });
 505        disabledSites = defaultSites_grouped_domains.concat(customSites_domains).filter(x => !enabledSites.includes(x));
 506        add_grouped_enabled_domains(grouped_sites);
 507        set_rules(sites, updatedSites, customSites);
 508      }
 509      if (key === 'sites_custom') {
 510        var sites_custom = storageChange.newValue ? storageChange.newValue : {};
 511        var sites_custom_old = storageChange.oldValue ? storageChange.oldValue : {};
 512        customSites = sites_custom;
 513        customSites_domains = Object.values(sites_custom).map(x => x.domain);
 514        
 515        // add/remove custom sites in options (not for default site(group))
 516        var sites_custom_added = Object.keys(sites_custom).filter(x => !Object.keys(sites_custom_old).includes(x) && !defaultSites.hasOwnProperty(x) && !defaultSites_domains.includes(sites_custom[x].domain));
 517        var sites_custom_removed = Object.keys(sites_custom_old).filter(x => !Object.keys(sites_custom).includes(x) && !defaultSites.hasOwnProperty(x) && !defaultSites_domains.includes(sites_custom_old[x].domain));
 518        
 519        ext_api.storage.local.get({
 520          sites: {}
 521        }, function (items) {
 522          var sites = items.sites;
 523          if (sites_custom_added.concat(sites_custom_removed).length > 0) {
 524            for (let key of sites_custom_added)
 525              sites[key] = sites_custom[key].domain;
 526            for (let key of sites_custom_removed)
 527              delete sites[key];
 528            
 529            ext_api.storage.local.set({
 530              sites: sites
 531            }, function () {
 532              true;
 533            });
 534          } else
 535            set_rules(sites, updatedSites, customSites);
 536        });
 537      }
 538      if (key === 'sites_updated') {
 539        var sites_updated = storageChange.newValue ? storageChange.newValue : {};
 540        updatedSites = sites_updated;
 541        updatedSites_domains_new = Object.values(updatedSites).filter(x => (x.domain && !defaultSites_domains.includes(x.domain) || x.group)).map(x => x.group ? x.group.filter(y => !defaultSites_domains.includes(y)) : x.domain).flat();
 542        updatedSites_new = Object.keys(updatedSites).filter(x => updatedSites[x].domain && !defaultSites_domains.includes(updatedSites[x].domain) && updatedSites[x].domain !== '###_usa_theathletic');
 543        if (updatedSites_new.length > 0) {
 544          if (enabledSites.includes('#options_enable_new_sites')) {
 545            for (let site_updated_new of updatedSites_new)
 546              optionSites[site_updated_new] = updatedSites[site_updated_new].domain;
 547            ext_api.storage.local.set({
 548              sites: optionSites
 549            });
 550          }
 551        } else
 552          set_rules(optionSites, updatedSites, customSites);
 553      }
 554      if (key === 'sites_excluded') {
 555        var sites_excluded = storageChange.newValue ? storageChange.newValue : [];
 556        var sites_excluded_old = storageChange.oldValue ? storageChange.oldValue : [];
 557        excludedSites = sites_excluded;
 558  
 559        // add/remove excluded sites in en/disabledSites
 560        var sites_excluded_added = sites_excluded.filter(x => !sites_excluded_old.includes(x));
 561        var sites_excluded_removed = sites_excluded_old.filter(x => !sites_excluded.includes(x));
 562  
 563        for (let site of sites_excluded_added) {
 564          if (enabledSites.includes(site)) {
 565            enabledSites.splice(enabledSites.indexOf(site), 1);
 566            disabledSites.push(site);
 567          }
 568        }
 569        for (let site of sites_excluded_removed) {
 570          if (disabledSites.includes(site)) {
 571            disabledSites.splice(disabledSites.indexOf(site), 1);
 572            enabledSites.push(site);
 573          }
 574        }
 575      }
 576      if (key === 'ext_version_new') {
 577        ext_version_new = storageChange.newValue;
 578      }
 579      if (key === 'optIn') {
 580        optin_setcookie = storageChange.newValue;
 581      }
 582      if (key === 'optInUpdate') {
 583        optin_update = storageChange.newValue;
 584      }
 585    }
 586  });
 587  
 588  // Set and show default options on install
 589  ext_api.runtime.onInstalled.addListener(function (details) {
 590    if (details.reason == "install") {
 591      setDefaultOptions();
 592    } else if (details.reason == "update") {
 593      ext_api.management.getSelf(function (result) {
 594        if (enabledSites.includes('#options_on_update') && result.installType !== 'development')
 595          ext_api.runtime.openOptionsPage(); // User updated extension (non-developer mode)
 596      });
 597    }
 598  });
 599  
 600  // Google AMP cache redirect
 601  ext_api.webRequest.onBeforeRequest.addListener(function (details) {
 602    var url = details.url.split('?')[0];
 603    var updatedUrl;
 604    if (matchUrlDomain('cdn.ampproject.org', url))
 605      updatedUrl = 'https://' + url.split(/cdn\.ampproject\.org\/[a-z]\/s\//)[1];
 606    else if (matchUrlDomain('google.com', url))
 607      updatedUrl = 'https://' + url.split(/\.google\.com\/amp\/s\//)[1];
 608    return { redirectUrl: decodeURIComponent(updatedUrl) };
 609  },
 610  {urls:["*://*.cdn.ampproject.org/*/s/*", "*://*.google.com/amp/s/*"], types:["main_frame"]},
 611  ["blocking"]
 612  );
 613  
 614  // inkl bypass
 615  ext_api.webRequest.onBeforeRequest.addListener(function (details) {
 616    if (!isSiteEnabled(details)) {
 617      return;
 618    }
 619    var updatedUrl = details.url.replace(/etok=[\w]*&/, '');
 620    if (details.url.includes('/signin?') && details.url.includes('redirect_to='))
 621      updatedUrl = 'https://www.inkl.com' + decodeURIComponent(updatedUrl.split('redirect_to=')[1]);
 622    return { redirectUrl: updatedUrl };
 623  },
 624  {urls:["*://*.inkl.com/*"], types:["main_frame"]},
 625  ["blocking"]
 626  );
 627  
 628  // m.faz.net set user-agent to mobile
 629  const userAgentMobile = "Mozilla/5.0 (Linux; Android 12) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Mobile Safari/537.36";
 630  ext_api.webRequest.onBeforeSendHeaders.addListener(function (details) {
 631    if (!isSiteEnabled(details)) {
 632      return;
 633    }
 634    let headers = details.requestHeaders;
 635    headers = headers.map(function (header) {
 636      if (header.name.toLowerCase() === 'user-agent')
 637        header.value = userAgentMobile;
 638      return header;
 639    });
 640    return {
 641      requestHeaders: headers
 642    };
 643  }, {
 644    urls: ["*://m.faz.net/*"],
 645    types: ["xmlhttprequest"]
 646  },
 647    ["blocking", "requestHeaders"]);
 648  
 649  // webcache.googleusercontent.com set user-agent to Chrome (on Firefox for Android)
 650  if ((typeof browser !== 'object') && navigator_ua_mobile) {
 651    ext_api.webRequest.onBeforeSendHeaders.addListener(function (details) {
 652      let headers = details.requestHeaders;
 653      headers = headers.map(function (header) {
 654        if (header.name.toLowerCase() === 'user-agent')
 655          header.value = userAgentMobile;
 656        return header;
 657      });
 658      return {
 659        requestHeaders: headers
 660      };
 661    }, {
 662      urls: ["*://webcache.googleusercontent.com/*"],
 663      types: ["main_frame", "xmlhttprequest"]
 664    },
 665      ["blocking", "requestHeaders"]);
 666  }
 667  
 668  // economictimes redirect
 669  ext_api.webRequest.onBeforeRequest.addListener(function (details) {
 670    if (!isSiteEnabled(details) || details.url.includes('.com/epaper/') || !navigator_ua_mobile) {
 671      return;
 672    }
 673    var updatedUrl = details.url.split('?')[0].replace('economictimes.indiatimes.com', 'm.economictimes.com');
 674    return { redirectUrl: updatedUrl };
 675  },
 676  {urls:["*://economictimes.indiatimes.com/*?from=mdr"], types:["main_frame"]},
 677  ["blocking"]
 678  );
 679  
 680  // infzm.com redirect to wap (mobile)
 681  ext_api.webRequest.onBeforeRequest.addListener(function (details) {
 682    if (!isSiteEnabled(details)) {
 683      return;
 684    }
 685    var updatedUrl = details.url.replace('.com/contents/', '.com/wap/#/content/');
 686    return { redirectUrl: updatedUrl };
 687  },
 688  {urls:["*://www.infzm.com/contents/*"], types:["main_frame"]},
 689  ["blocking"]
 690  );
 691  
 692  // telegraaf.nl redirect error-page
 693  ext_api.webRequest.onBeforeRequest.addListener(function (details) {
 694    if (!isSiteEnabled(details)) {
 695      return;
 696    }
 697    let updatedUrl = details.url.split('&')[0].replace('error?ref=/', '');;
 698    return { redirectUrl: updatedUrl };
 699  },
 700  {urls:["*://www.telegraaf.nl/error?ref=/*"], types:["main_frame"]},
 701  ["blocking"]
 702  );
 703  
 704  // Australia News Corp redirect subscribe to amp
 705  var au_news_corp_no_amp_fix = ['codesports.com.au'];
 706  var au_news_corp_subscr = au_news_corp_domains.filter(domain => !au_news_corp_no_amp_fix.includes(domain)).map(domain => '*://www.' + domain + '/subscribe/*');
 707  ext_api.webRequest.onBeforeRequest.addListener(function (details) {
 708    if (!isSiteEnabled(details) || details.url.includes('/digitalprinteditions') || !(details.url.includes('dest=') && details.url.split('dest=')[1].split('&')[0])) {
 709      return;
 710    }
 711    var updatedUrl = decodeURIComponent(details.url.split('dest=')[1].split('&')[0]) + '?amp';
 712    return {
 713      redirectUrl: updatedUrl
 714    };
 715  }, {
 716    urls: au_news_corp_subscr,
 717    types: ["main_frame"]
 718  },
 719    ["blocking"]);
 720  
 721  // fix nytimes x-frame-options (hidden iframe content)
 722  ext_api.webRequest.onHeadersReceived.addListener(function (details) {
 723    if (!isSiteEnabled(details)) {
 724      return;
 725    }
 726    var headers = details.responseHeaders;
 727    headers = headers.map(function (header) {
 728        if (header.name === 'x-frame-options')
 729          header.value = 'SAMEORIGIN';
 730        return header;
 731      });
 732    return {
 733      responseHeaders: headers
 734    };
 735  }, {
 736    urls: ["*://*.nytimes.com/*"]
 737  },
 738    ['blocking', 'responseHeaders']);
 739  
 740  function blockJsInlineListener(details) {
 741    let domain = matchUrlDomain(blockedJsInlineDomains, details.url);
 742    let matched = domain && details.url.match(blockedJsInline[domain]);
 743    if (matched && optin_setcookie && ['uol.com.br'].includes(domain))
 744      matched = false;
 745    if (!isSiteEnabled(details) || !matched)
 746      return;
 747    var headers = details.responseHeaders;
 748    headers.push({
 749      'name': 'Content-Security-Policy',
 750      'value': "script-src *;"
 751    });
 752    return {
 753      responseHeaders: headers
 754    };
 755  }
 756  
 757  function disableJavascriptInline() {
 758    // block inline script
 759    ext_api.webRequest.onHeadersReceived.removeListener(blockJsInlineListener);
 760    var block_js_inline_urls = [];
 761    for (let domain in blockedJsInline)
 762      block_js_inline_urls.push("*://*." + domain + "/*");
 763    if (block_js_inline_urls.length)
 764      ext_api.webRequest.onHeadersReceived.addListener(blockJsInlineListener, {
 765        'types': ['main_frame', 'sub_frame'],
 766        'urls': block_js_inline_urls
 767      },
 768        ['blocking', 'responseHeaders']);
 769  }
 770  
 771  if (typeof browser !== 'object') {
 772    var focus_changed = false;
 773    ext_api.windows.onFocusChanged.addListener((windowId) => {
 774      if (windowId > 0)
 775        focus_changed = true;
 776    });
 777  }
 778  
 779    function runOnTab(tab) {
 780      let tabId = tab.id;
 781      let url = tab.url;
 782      let rc_domain = matchUrlDomain(remove_cookies, url);
 783      let rc_domain_enabled = rc_domain && enabledSites.includes(rc_domain);
 784      let lib_file = 'lib/empty.js';
 785      if (matchUrlDomain(dompurify_sites, url))
 786        lib_file = 'lib/purify.min.js';
 787      var bg2csData = {};
 788      if (optin_setcookie && matchUrlDomain(['###'], url))
 789        bg2csData.optin_setcookie = 1;
 790      if (matchUrlDomain(amp_unhide, url))
 791        bg2csData.amp_unhide = 1;
 792      let amp_redirect_domain = '';
 793      if (amp_redirect_domain = matchUrlDomain(Object.keys(amp_redirect), url))
 794        bg2csData.amp_redirect = amp_redirect[amp_redirect_domain];
 795      let cs_code_domain = '';
 796      if (cs_code_domain = matchUrlDomain(Object.keys(cs_code), url))
 797        bg2csData.cs_code = cs_code[cs_code_domain];
 798      let ld_json_domain = '';
 799      if (ld_json_domain = matchUrlDomain(Object.keys(ld_json), url))
 800        bg2csData.ld_json = ld_json[ld_json_domain];
 801      let ld_json_next_domain = '';
 802      if (ld_json_next_domain = matchUrlDomain(Object.keys(ld_json_next), url))
 803        bg2csData.ld_json_next = ld_json_next[ld_json_next_domain];
 804      let ld_google_webcache_domain = '';
 805      if (ld_google_webcache_domain = matchUrlDomain(Object.keys(ld_google_webcache), url))
 806        bg2csData.ld_google_webcache = ld_google_webcache[ld_google_webcache_domain];
 807      let add_ext_link_domain = '';
 808      if (add_ext_link_domain = matchUrlDomain(Object.keys(add_ext_link), url))
 809        bg2csData.add_ext_link = add_ext_link[add_ext_link_domain];
 810      let tab_runs = 5;
 811      for (let n = 0; n < tab_runs; n++) {
 812        setTimeout(function () {
 813          // run contentScript.js on page
 814          ext_api.tabs.executeScript(tabId, {
 815            file: lib_file,
 816            runAt: 'document_start'
 817          }, function (res) {
 818            if (ext_api.runtime.lastError)
 819              return;
 820            ext_api.tabs.executeScript(tabId, {
 821              file: 'contentScript.js',
 822              runAt: 'document_start'
 823            }, function (res) {
 824              if (ext_api.runtime.lastError || res[0]) {
 825                return;
 826              }
 827            })
 828          });
 829          // send bg2csData to contentScript.js
 830          if (Object.keys(bg2csData).length) {
 831            setTimeout(function () {
 832              ext_api.tabs.sendMessage(tabId, {msg: "bg2cs", data: bg2csData});
 833            }, 500);
 834          }
 835          // remove cookies after page load
 836          if (rc_domain_enabled) {
 837            remove_cookies_fn(rc_domain, true);
 838          }
 839        }, n * 200);
 840      }
 841    }
 842  
 843    function runOnTab_once(tab) {
 844      let tabId = tab.id;
 845      let url = tab.url;
 846      // load contentScript_once.js to identify custom site (flex) of group
 847      if (!(matchUrlDomain(custom_flex_domains.concat(custom_flex_not_domains, customSites_domains, updatedSites_domains_new, excludedSites, nofix_sites), url) || matchUrlDomain(defaultSites_domains, url))) {
 848        ext_api.tabs.executeScript(tabId, {
 849          file: 'contentScript_once.js',
 850          runAt: 'document_start'
 851        }, function (res) {
 852          if (ext_api.runtime.lastError || res[0]) {
 853            return;
 854          }
 855        });
 856      }
 857      // load toggleIcon.js (icon for dark or incognito mode in Chrome))
 858      if (typeof browser !== 'object') {
 859        ext_api.tabs.executeScript(tabId, {
 860          file: 'options/toggleIcon.js',
 861          runAt: 'document_start'
 862        }, function (res) {
 863          if (ext_api.runtime.lastError || res[0]) {
 864            return;
 865          }
 866        });
 867      }
 868    }
 869  
 870    var set_var_sites =  ['nzherald.co.nz', 'theglobeandmail.com'].concat(de_madsack_domains);
 871    function runOnTab_once_var(tab) {
 872      let tabId = tab.id;
 873      let url = tab.url;
 874      let domain = matchUrlDomain(set_var_sites, url);
 875      // load contentScript_once_var.js to set variables for site
 876      if (domain && enabledSites.includes(domain)) {
 877        ext_api.tabs.executeScript(tabId, {
 878          file: 'contentScript_once_var.js',
 879          runAt: 'document_start'
 880        }, function (res) {
 881          if (ext_api.runtime.lastError || res[0]) {
 882            return;
 883          }
 884        });
 885      }
 886    }
 887  
 888  ext_api.tabs.onUpdated.addListener((tabId, changeInfo, tab) => {
 889    let tab_status = changeInfo.status;
 890    if (/^http/.test(tab.url)) {
 891      if ((tab_status && tab_status === 'complete') || (changeInfo.url)) {
 892        let timeout = changeInfo.url ? 500 : 0;
 893        setTimeout(function () {
 894          if (matchUrlDomain(enabledSites, tab.url)) {
 895            runOnTab(tab);
 896          }
 897          runOnTab_once(tab);
 898        }, timeout);
 899      }
 900      runOnTab_once_var(tab);
 901    }
 902  });
 903  
 904  setInterval(function () {
 905    let current_date_str = currentDateStr();
 906    if (last_date_str < current_date_str) {
 907      bpc_count_daily_users(current_date_str);
 908      last_date_str = current_date_str;
 909    }
 910  }, 60 * 60 * 1000);
 911  
 912  var extraInfoSpec = ['blocking', 'requestHeaders'];
 913  if (ext_api.webRequest.OnBeforeSendHeadersOptions.hasOwnProperty('EXTRA_HEADERS'))
 914    extraInfoSpec.push('extraHeaders');
 915  
 916  ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
 917    var requestHeaders = details.requestHeaders;
 918  
 919    var header_referer = '';
 920    if (details.originUrl)
 921      header_referer = details.originUrl;
 922    else {
 923      for (let n in requestHeaders) {
 924        if (requestHeaders[n].name.toLowerCase() == 'referer') {
 925          header_referer = requestHeaders[n].value;
 926          break;
 927        }
 928      }
 929      var blocked_referer_domains = ['timeshighereducation.com'];
 930      if (!header_referer && details.initiator) {
 931        header_referer = details.initiator;
 932        if (!blocked_referer && matchUrlDomain(blocked_referer_domains, details.url) && ['script', 'xmlhttprequest'].includes(details.type)) {
 933          for (let domain of blocked_referer_domains)
 934            restrictions[domain] = new RegExp('((\\/|\\.)' + domain.replace(/\./g, '\\.') + '($|\\/$)|' + restrictions[domain].toString().replace(/(^\/|\/$)/g, '') + ')');
 935          blocked_referer = true;
 936        }
 937      }
 938    }
 939  
 940    // block external javascript for custom sites (optional)
 941    if (['script'].includes(details.type)) {
 942      let domain_blockjs_ext = matchUrlDomain(block_js_custom_ext, header_referer);
 943      if (domain_blockjs_ext && !matchUrlDomain(domain_blockjs_ext, details.url) && isSiteEnabled({url: header_referer}))
 944        return { cancel: true };
 945    }
 946  
 947    // check for blocked regular expression: domain enabled, match regex, block on an internal or external regex
 948    if (['script', 'xmlhttprequest'].includes(details.type)) {
 949      let domain = matchUrlDomain(blockedRegexesDomains, header_referer);
 950      if (domain && details.url.match(blockedRegexes[domain]) && isSiteEnabled({url: header_referer}))
 951        return { cancel: true };
 952    }
 953  
 954    // block general paywall scripts
 955    if (['script', 'xmlhttprequest'].includes(details.type)) {
 956      for (let domain in blockedRegexesGeneral) {
 957        if (details.url.match(blockedRegexesGeneral[domain].block_regex) && !(matchUrlDomain(excludedSites.concat(disabledSites, blockedRegexesGeneral[domain].excluded_domains), header_referer)))
 958          return { cancel: true };
 959      }
 960    }
 961  
 962    if (!isSiteEnabled(details)) {
 963      return;
 964    }
 965  
 966    // block javascript of (sub)domain for custom sites (optional)
 967    var domain_blockjs = matchUrlDomain(block_js_custom, details.url);
 968    if (domain_blockjs && matchUrlDomain(domain_blockjs, details.url) && details.type === 'script') {
 969      return { cancel: true };
 970    }
 971  
 972    var useUserAgentMobile = false;
 973    var setReferer = false;
 974  
 975  var ignore_types = ['font', 'image', 'stylesheet'];
 976  if (matchUrlDomain(au_news_corp_domains, details.url))
 977    ignore_types = ['font', 'image', 'stylesheet', 'other', 'script', 'xmlhttprequest'];
 978  
 979  if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(details.type)) {
 980    var mobile = details.requestHeaders.filter(x => x.name.toLowerCase() === "user-agent" && x.value.toLowerCase().includes("mobile")).length;
 981    var googlebotEnabled = matchUrlDomain(use_google_bot, details.url) && 
 982      !(matchUrlDomain(es_grupo_vocento_domains, details.url) && mobile) &&
 983      !(matchUrlDomain('barrons.com', details.url) && enabledSites.includes('#options_disable_gb_barrons')) &&
 984      !(matchUrlDomain(['economictimes.com', 'economictimes.indiatimes.com'], details.url) && !details.url.split(/\?|#/)[0].endsWith('.cms')) &&
 985      !(matchUrlDomain(au_news_corp_domains, details.url) && (details.url.includes('?amp') || (!matchUrlDomain(au_news_corp_no_amp_fix, details.url) && enabledSites.includes('#options_disable_gb_au_news_corp')))) &&
 986      !(matchUrlDomain('uol.com.br', details.url) && !matchUrlDomain('folha.uol.com.br', details.url)) &&
 987      !(matchUrlDomain('wsj.com', details.url) && (enabledSites.includes('#options_disable_gb_wsj') || (!details.url.includes('/articles/') && details.type === 'main_frame' && mobile)));
 988    var bingbotEnabled = matchUrlDomain(use_bing_bot, details.url);
 989    var facebookbotEnabled = matchUrlDomain(use_facebook_bot, details.url);
 990  
 991    // if referer exists, set it
 992    requestHeaders = requestHeaders.map(function (requestHeader) {
 993      if (requestHeader.name === 'Referer') {
 994        if (googlebotEnabled || matchUrlDomain(use_google_referer, details.url)) {
 995          requestHeader.value = 'https://www.google.com/';
 996        } else if (matchUrlDomain(use_facebook_referer, details.url)) {
 997          requestHeader.value = 'https://www.facebook.com/';
 998        } else if (matchUrlDomain(use_twitter_referer, details.url)) {
 999          requestHeader.value = 'https://t.co/';
1000        }
1001        setReferer = true;
1002      }
1003      if (requestHeader.name === 'User-Agent') {
1004        useUserAgentMobile = requestHeader.value.toLowerCase().includes("mobile") && !matchUrlDomain(['telerama.fr'], details.url);
1005      }
1006      return requestHeader;
1007    });
1008  
1009    // otherwise add it
1010    if (!setReferer) {
1011      if (googlebotEnabled || matchUrlDomain(use_google_referer, details.url)) {
1012        requestHeaders.push({
1013          name: 'Referer',
1014          value: 'https://www.google.com/'
1015        });
1016      } else if (matchUrlDomain(use_facebook_referer, details.url)) {
1017        requestHeaders.push({
1018          name: 'Referer',
1019          value: 'https://www.facebook.com/'
1020        });
1021      } else if (matchUrlDomain(use_twitter_referer, details.url)) {
1022        requestHeaders.push({
1023          name: 'Referer',
1024          value: 'https://t.co/'
1025        });
1026      }
1027    }
1028  
1029    // override User-Agent to use Googlebot
1030    if (googlebotEnabled) {
1031      requestHeaders.push({
1032        "name": "User-Agent",
1033        "value": useUserAgentMobile ? userAgentMobileG : userAgentDesktopG
1034      })
1035      requestHeaders.push({
1036        "name": "X-Forwarded-For",
1037        "value": "66.249.66.1"
1038      })
1039    }
1040  
1041    // override User-Agent to use Bingbot
1042    if (bingbotEnabled) {
1043      requestHeaders.push({
1044        "name": "User-Agent",
1045        "value": useUserAgentMobile ? userAgentMobileB : userAgentDesktopB
1046      })
1047    }
1048  
1049    // override User-Agent to use Facebookbot
1050    if (facebookbotEnabled) {
1051      requestHeaders.push({
1052        "name": "User-Agent",
1053        "value": userAgentDesktopF
1054      })
1055    }
1056  
1057    // random IP for sites in use_random_ip
1058    let domain_random;
1059    if (domain_random = matchUrlDomain(use_random_ip, details.url)) {
1060      let randomIP_val;
1061      if (random_ip[domain_random] === 'eu')
1062        randomIP_val = randomIP(185, 185);
1063      else
1064        randomIP_val = randomIP();
1065      requestHeaders.push({
1066        "name": "X-Forwarded-For",
1067        "value": randomIP_val
1068      })
1069    }
1070  }
1071  
1072    // remove cookies before page load
1073    if (!matchUrlDomain(allow_cookies, details.url)) {
1074      requestHeaders = requestHeaders.map(function(requestHeader) {
1075        if (requestHeader.name === 'Cookie') {
1076          requestHeader.value = '';
1077        }
1078        return requestHeader;
1079      });
1080    }
1081  
1082    if (kiwi_browser) {
1083      let tabId = details.tabId;
1084      if (tabId !== -1) {
1085        if (['main_frame', 'sub_frame', 'xmlhttprequest'].includes(details.type)) {
1086          ext_api.tabs.get(tabId, function (tab) {
1087            if (!ext_api.runtime.lastError && tab && isSiteEnabled(tab)) {
1088              runOnTab(tab);
1089            }
1090            runOnTab_once(tab);
1091            runOnTab_once_var(tab);
1092          });
1093        }
1094      } else {
1095        if (['xmlhttprequest'].includes(details.type)) {
1096          ext_api.tabs.query({
1097            active: true,
1098            currentWindow: true
1099          }, function (tabs) {
1100            if (tabs && tabs[0] && /^http/.test(tabs[0].url)) {
1101              let tab = tabs[0];
1102              if (isSiteEnabled(tab)) {
1103                runOnTab(tab);
1104              }
1105              runOnTab_once(tab);
1106              runOnTab_once_var(tab);
1107            }
1108          });
1109        }
1110      }
1111    }
1112  
1113    return { requestHeaders: requestHeaders };
1114  }, {
1115    urls: ['*://*/*']
1116  }, extraInfoSpec);
1117  // extraInfoSpec is ['blocking', 'requestHeaders'] + possible 'extraHeaders'
1118  
1119  function check_sites_custom_ext() {
1120    fetch(sites_custom_ext_json)
1121    .then(response => {
1122      if (response.ok) {
1123        response.json().then(json => {
1124          customSitesExt = Object.values(json).map(x => x.domain);
1125        })
1126      }
1127    }).catch(function (err) {
1128      false;
1129    });
1130  }
1131  
1132  var customSitesExt = [];
1133  var sites_custom_ext_json = 'custom/sites_custom.json';
1134  
1135  ext_api.tabs.onUpdated.addListener(function (tabId, info, tab) { updateBadge(tab); });
1136  ext_api.tabs.onActivated.addListener(function (activeInfo) { if (activeInfo.tabId) ext_api.tabs.get(activeInfo.tabId, updateBadge); });
1137  
1138  function updateBadge(activeTab) {
1139    if (ext_api.runtime.lastError || !activeTab || (activeTab.url && urlHost(activeTab.url).match(/^archive\.[a-z]{2}$/)))
1140      return;
1141    let badgeText = '';
1142    let color = 'red';
1143    let currentUrl = activeTab.url;
1144    if (currentUrl) {
1145      if (isSiteEnabled({url: currentUrl})) {
1146        badgeText = 'ON';
1147        color = 'red';
1148      } else if (matchUrlDomain(enabledSites, currentUrl)) {
1149        badgeText = 'ON-';
1150        color = 'orange';
1151      } else if (matchUrlDomain(disabledSites, currentUrl)) {
1152        badgeText = 'OFF';
1153        color = 'blue';
1154      } else if (matchUrlDomain(nofix_sites, currentUrl)) {
1155        badgeText = 'X';
1156        color = 'silver';
1157      }
1158      if (matchUrlDomain('webcache.googleusercontent.com', currentUrl))
1159        badgeText = '';
1160      if (ext_version_new)
1161        badgeText = '^' + badgeText;
1162      let isDefaultSite = matchUrlDomain(defaultSites_domains, currentUrl);
1163      let isCustomSite = matchUrlDomain(customSites_domains, currentUrl);
1164      let isUpdatedSite = matchUrlDomain(updatedSites_domains_new, currentUrl);
1165      if (!isDefaultSite && (isCustomSite || isUpdatedSite)) {
1166        ext_api.permissions.contains({
1167          origins: ['*://*.' + (isCustomSite || isUpdatedSite) + '/*']
1168        }, function (result) {
1169          if (!result)
1170            badgeText = enabledSites.includes(isCustomSite || isUpdatedSite) ? 'C' : '';
1171          if (color && badgeText)
1172            ext_api.action.setBadgeBackgroundColor({color: color});
1173          ext_api.action.setBadgeText({text: badgeText});
1174        });
1175      } else {
1176        if (!badgeText && matchUrlDomain(customSitesExt, currentUrl))
1177          badgeText = '+C';
1178        if (color && badgeText)
1179          ext_api.action.setBadgeBackgroundColor({color: color});
1180        ext_api.action.setBadgeText({text: badgeText});
1181      }
1182    } else
1183        ext_api.action.setBadgeText({text: badgeText});
1184  }
1185  
1186  var ext_version_new;
1187  function check_update() {
1188    let manifest_new = 'https://gitlab.com/magnolia1234/bypass-paywalls-' + url_loc + '-clean/raw/master/manifest.json';
1189    fetch(manifest_new)
1190    .then(response => {
1191      if (response.ok) {
1192        response.json().then(json => {
1193          ext_api.management.getSelf(function (result) {
1194            var installType = result.installType;
1195            var ext_version_len = (installType === 'development') ? 7 : 5;
1196            ext_version_new = json['version'];
1197            if (ext_version_new.substring(0, ext_version_len) <= ext_version.substring(0, ext_version_len))
1198              ext_version_new = '';
1199            ext_api.storage.local.set({
1200              ext_version_new: ext_version_new
1201            });
1202          });
1203        })
1204      }
1205    }).catch(function (err) {
1206      false;
1207    });
1208  }
1209  
1210  function site_switch() {
1211    ext_api.tabs.query({
1212      active: true,
1213      currentWindow: true
1214    }, function (tabs) {
1215      if (tabs && tabs[0] && /^http/.test(tabs[0].url)) {
1216        let currentUrl = tabs[0].url;
1217        let isDefaultSite = matchUrlDomain(defaultSites_grouped_domains, currentUrl);
1218        if (!isDefaultSite) {
1219          let isDefaultSiteGroup = matchUrlDomain(defaultSites_domains, currentUrl);
1220          if (isDefaultSiteGroup)
1221            isDefaultSite = Object.keys(grouped_sites).find(key => grouped_sites[key].includes(isDefaultSiteGroup));
1222        }
1223        if (!isDefaultSite) {
1224          let sites_updated_domains_new = Object.values(updatedSites).filter(x => x.domain && !defaultSites_domains.includes(x.domain));
1225          let isUpdatedSite = matchUrlDomain(sites_updated_domains_new, currentUrl);
1226          if (isUpdatedSite)
1227            isDefaultSite = isUpdatedSite;
1228        }
1229        let defaultSite_title = isDefaultSite ? Object.keys(defaultSites).find(key => defaultSites[key].domain === isDefaultSite) : '';
1230        let isCustomSite = matchUrlDomain(Object.values(customSites_domains), currentUrl);
1231        let customSite_title = isCustomSite ? Object.keys(customSites).find(key => customSites[key].domain === isCustomSite) : '';
1232        let site_title = defaultSite_title || customSite_title;
1233        let domain = isDefaultSite || isCustomSite;
1234        if (domain && site_title) {
1235          let added_site = [];
1236          let removed_site = [];
1237          if (enabledSites.includes(domain))
1238            removed_site.push(site_title);
1239          else
1240            added_site.push(site_title);
1241          ext_api.storage.local.get({
1242            sites: {}
1243          }, function (items) {
1244            var sites = items.sites;
1245            for (let key of added_site)
1246              sites[key] = domain;
1247            for (let key of removed_site) {
1248              key = Object.keys(sites).find(sites_key => compareKey(sites_key, key));
1249              delete sites[key];
1250            }
1251            ext_api.storage.local.set({
1252              sites: sites
1253            }, function () {
1254              ext_api.tabs.reload({bypassCache: true});
1255            });
1256          });
1257        }
1258      }
1259    });
1260  }
1261  
1262  function remove_cookies_fn(domainVar, exclusions = false) {
1263    ext_api.cookies.getAllCookieStores(function (cookieStores) {
1264      ext_api.tabs.query({
1265        active: true,
1266        currentWindow: true
1267      }, function (tabs) {
1268        if (!ext_api.runtime.lastError && tabs && tabs[0] && /^http/.test(tabs[0].url)) {
1269          let tabId = tabs[0].id;
1270          let storeId = '0';
1271          for (let store of cookieStores) {
1272            if (store.tabIds.includes(tabId))
1273              storeId = store.id;
1274          }
1275          storeId = storeId.toString();
1276          if (domainVar === 'asia.nikkei.com')
1277            domainVar = 'nikkei.com';
1278          var cookie_get_options = {
1279            domain: domainVar
1280          };
1281          if (storeId !== 'null')
1282            cookie_get_options.storeId = storeId;
1283          var cookie_remove_options = {};
1284          ext_api.cookies.getAll(cookie_get_options, function (cookies) {
1285            for (let cookie of cookies) {
1286              if (exclusions) {
1287                var rc_domain = cookie.domain.replace(/^(\.?www\.|\.)/, '');
1288                // hold specific cookie(s) from remove_cookies domains
1289                if ((rc_domain in remove_cookies_select_hold) && remove_cookies_select_hold[rc_domain].includes(cookie.name)) {
1290                  continue; // don't remove specific cookie
1291                }
1292                // drop only specific cookie(s) from remove_cookies domains
1293                if ((rc_domain in remove_cookies_select_drop) && !(remove_cookies_select_drop[rc_domain].includes(cookie.name))) {
1294                  continue; // only remove specific cookie
1295                }
1296                // hold on to consent-cookie
1297                if (cookie.name.match(/(consent|^optanon)/i)) {
1298                  continue;
1299                }
1300              }
1301              cookie.domain = cookie.domain.replace(/^\./, '');
1302              cookie_remove_options = {
1303                url: (cookie.secure ? "https://" : "http://") + cookie.domain + cookie.path,
1304                name: cookie.name
1305              };
1306              if (storeId !== 'null')
1307                cookie_remove_options.storeId = storeId;
1308              ext_api.cookies.remove(cookie_remove_options);
1309            }
1310          });
1311        }
1312      });
1313    })
1314  }
1315  
1316  function clear_cookies() {
1317    ext_api.tabs.query({
1318      active: true,
1319      currentWindow: true
1320    }, function (tabs) {
1321      if (tabs && tabs[0] && /^http/.test(tabs[0].url)) {
1322        ext_api.tabs.executeScript({
1323          file: 'options/clearCookies.js',
1324          runAt: 'document_start'
1325        }, function (res) {
1326          if (ext_api.runtime.lastError || res[0]) {
1327            return;
1328          }
1329        });
1330        ext_api.tabs.update(tabs[0].id, {
1331          url: tabs[0].url
1332        });
1333      }
1334    });
1335  }
1336  
1337  var chrome_scheme = 'light';
1338  ext_api.runtime.onMessage.addListener(function (message, sender) {
1339    if (message.request === 'clear_cookies') {
1340      clear_cookies();
1341    }
1342    // clear cookies for domain
1343    if (message.request === 'clear_cookies_domain' && message.data) {
1344      remove_cookies_fn(message.data.domain);
1345    }
1346    if (message.request === 'custom_domain' && message.data && message.data.domain) {
1347      let custom_domain = message.data.domain;
1348      let group = message.data.group;
1349      if (group) {
1350        let nofix_groups = ['###_be_mediahuis', '###_ch_tamedia', '###_de_rp_aachen_medien', '###_fi_alma_talent', '###_it_citynews', '###_nl_vmnmedia', '###_substack_custom'];
1351        if (!custom_flex_domains.includes(custom_domain)) {
1352          if (enabledSites.includes(group)) {
1353            let rules = Object.values(defaultSites).filter(x => x.domain === group)[0];
1354            if (rules) {
1355              if (group === '###_de_madsack') {
1356                if (!set_var_sites.includes(custom_domain))
1357                  set_var_sites.push(custom_domain);
1358              } else if (group === '###_usa_townnews') {
1359                if (!dompurify_sites.includes(custom_domain))
1360                  dompurify_sites.push(custom_domain);
1361                if (['berkshireeagle.com'].includes(custom_domain))
1362                  rules.useragent = 'googlebot';
1363              }
1364            } else
1365              rules = Object.values(customSites).filter(x => x.domain === group)[0];
1366            if (rules) {
1367              custom_flex_domains.push(custom_domain);
1368              if (!enabledSites.includes(custom_domain))
1369                enabledSites.push(custom_domain);
1370              customFlexAddRules(custom_domain, rules);
1371            }
1372          } else if (disabledSites.includes(group))
1373            custom_flex_not_domains.push(custom_domain);
1374          else if (nofix_groups.includes(group))
1375            nofix_sites.push(custom_domain);
1376      }
1377    } else
1378      custom_flex_not_domains.push(custom_domain);
1379    }
1380    if (message.request === 'site_switch') {
1381      site_switch();
1382    }
1383    if (message.request === 'check_sites_updated') {
1384      check_sites_updated();
1385    }
1386    if (message.request === 'clear_sites_updated') {
1387      clear_sites_updated();
1388    }
1389    if (message.request === 'popup_show_toggle') {
1390      ext_api.tabs.query({
1391        active: true,
1392        currentWindow: true
1393      }, function (tabs) {
1394        if (tabs && tabs[0] && /^http/.test(tabs[0].url)) {
1395          let currentUrl = tabs[0].url;
1396          let domain;
1397          let isExcludedSite = matchUrlDomain(excludedSites, currentUrl);
1398          if (!isExcludedSite) {
1399            let isDefaultSite = matchUrlDomain(defaultSites_domains, currentUrl);
1400            let isCustomSite = matchUrlDomain(Object.values(customSites_domains), currentUrl);
1401            domain = isDefaultSite || isCustomSite;
1402            if (domain)
1403              ext_api.runtime.sendMessage({
1404                msg: "popup_show_toggle",
1405                data: {
1406                  domain: domain,
1407                  enabled: enabledSites.includes(domain)
1408                }
1409              });
1410          }
1411        }
1412      });
1413    }
1414    if (message.request === 'refreshCurrentTab') {
1415      ext_api.tabs.reload({bypassCache: true});
1416    }
1417    if (message.request === 'getExtSrc' && message.data) {
1418      fetch(message.data.url)
1419      .then(response => {
1420        if (response.ok) {
1421          response.text().then(html => {
1422            if (message.data.base64) {
1423              html = decode_utf8(atob(html));
1424              message.data.selector_source = 'body';
1425            }
1426            message.data.html = html;
1427            if (typeof DOMParser === 'function') {
1428              let parser = new DOMParser();
1429              let doc = parser.parseFromString(html, 'text/html');
1430              let article_new = doc.querySelector(message.data.selector_source);
1431              if (article_new)
1432                message.data.html = article_new.outerHTML;
1433            }
1434            ext_api.tabs.query({
1435              active: true,
1436              currentWindow: true
1437            }, function (tabs) {
1438              if (tabs && tabs[0] && /^http/.test(tabs[0].url)) {
1439                ext_api.tabs.sendMessage(sender.tab.id, {msg: "showExtSrc", data: message.data});
1440              }
1441            });
1442          });
1443        }
1444      }).catch(function (err) {
1445        message.data.html = '';
1446        ext_api.tabs.query({
1447          active: true,
1448          currentWindow: true
1449        }, function (tabs) {
1450          if (tabs && tabs[0] && /^http/.test(tabs[0].url)) {
1451            ext_api.tabs.sendMessage(sender.tab.id, {msg: "showExtSrc", data: message.data});
1452          }
1453        });
1454      });
1455    }
1456    if (message.scheme && (![chrome_scheme, 'undefined'].includes(message.scheme) || focus_changed)) {
1457        let icon_path = {path: {'128': 'bypass.png'}};
1458        if (message.scheme === 'dark')
1459            icon_path = {path: {'128': 'bypass-dark.png'}};
1460        ext_api.action.setIcon(icon_path);
1461        chrome_scheme = message.scheme;
1462        focus_changed = false;
1463    }
1464  });
1465  
1466  // show the opt-in tab on installation
1467  ext_api.storage.local.get(["optInShown", "customShown"], function (result) {
1468    if (!result.optInShown || !result.customShown) {
1469      ext_api.tabs.create({
1470        url: "options/optin/opt-in.html"
1471      });
1472      ext_api.storage.local.set({
1473        "optInShown": true,
1474        "customShown": true
1475      });
1476    }
1477  });
1478  
1479  function filterObject(obj, filterFn, mapFn = function (val, key) {
1480    return [key, val];
1481  }) {
1482    return Object.fromEntries(Object.entries(obj).
1483      filter(([key, val]) => filterFn(val, key)).map(([key, val]) => mapFn(val, key)));
1484  }
1485  
1486  function compareKey(firstStr, secondStr) {
1487    return firstStr.toLowerCase().replace(/\s\(.*\)/, '') === secondStr.toLowerCase().replace(/\s\(.*\)/, '');
1488  }
1489  
1490  function isSiteEnabled(details) {
1491    var enabledSite = matchUrlDomain(enabledSites, details.url);
1492    if (!ext_name.startsWith('Bypass Paywalls Clean'))
1493      enabledSite = '';
1494    if (enabledSite in restrictions) {
1495      return restrictions[enabledSite].test(details.url);
1496    }
1497    return !!enabledSite;
1498  }
1499  
1500  function matchDomain(domains, hostname = '') {
1501    var matched_domain = false;
1502    if (typeof domains === 'string')
1503      domains = [domains];
1504    domains.some(domain => (hostname === domain || hostname.endsWith('.' + domain)) && (matched_domain = domain));
1505    return matched_domain;
1506  }
1507  
1508  function urlHost(url) {
1509    if (/^http/.test(url)) {
1510      try {
1511        return new URL(url).hostname;
1512      } catch (e) {
1513        console.log(`url not valid: ${url} error: ${e}`);
1514      }
1515    }
1516    return url;
1517  }
1518  
1519  function matchUrlDomain(domains, url) {
1520    return matchDomain(domains, urlHost(url));
1521  }
1522  
1523  function prepHostname(hostname) {
1524    return hostname.replace(/^(www|m|account|amp(\d)?|edition|eu|mobil|wap)\./, '');
1525  }
1526  
1527  function getParameterByName(name, url) {
1528    name = name.replace(/[\[\]]/g, '\\$&');
1529    var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'),
1530    results = regex.exec(url);
1531    if (!results) return null;
1532    if (!results[2]) return '';
1533    return decodeURIComponent(results[2].replace(/\+/g, ' '));
1534  }
1535  
1536  function stripQueryStringAndHashFromPath(url) {
1537    return url.split("?")[0].split("#")[0];
1538  }
1539  
1540  function decode_utf8(str) {
1541    return decodeURIComponent(escape(str));
1542  }
1543  
1544  function randomInt(max) {
1545    return Math.floor(Math.random() * Math.floor(max));
1546  }
1547  
1548  function randomIP(range_low = 0, range_high = 223) {
1549    let rndmIP = [];
1550    for (let n = 0; n < 4; n++) {
1551      if (n === 0)
1552        rndmIP.push(range_low + randomInt(range_high - range_low + 1));
1553      else
1554        rndmIP.push(randomInt(255) + 1);
1555    }
1556    return rndmIP.join('.');
1557  }
1558  
1559  // Refresh the current tab (http)
1560  function refreshCurrentTab() {
1561    ext_api.tabs.query({
1562      active: true,
1563      currentWindow: true
1564    }, function (tabs) {
1565      if (tabs && tabs[0] && /^http/.test(tabs[0].url)) {
1566        if (ext_api.runtime.lastError)
1567          return;
1568        ext_api.tabs.update(tabs[0].id, {
1569          url: tabs[0].url
1570        });
1571      }
1572    });
1573  }