1  /*
   2  * Copyright (c) 2013-2025, The PurpleI2P Project
   3  *
   4  * This file is part of Purple i2pd project and licensed under BSD3
   5  *
   6  * See full license text in LICENSE file at top of project tree
   7  */
   8  
   9  #include <string.h>
  10  #include <inttypes.h>
  11  #include <string>
  12  #include <unordered_map>
  13  #include <fstream>
  14  #include <chrono>
  15  #include <condition_variable>
  16  #include <openssl/rand.h>
  17  #include <boost/algorithm/string.hpp>
  18  #include "Base.h"
  19  #include "util.h"
  20  #include "Timestamp.h"
  21  #include "Identity.h"
  22  #include "FS.h"
  23  #include "Log.h"
  24  #include "HTTP.h"
  25  #include "NetDb.hpp"
  26  #include "ClientContext.h"
  27  #include "AddressBook.h"
  28  #include "Config.h"
  29  
  30  #if STD_FILESYSTEM
  31  #include <filesystem>
  32  namespace fs_lib = std::filesystem;
  33  #else
  34  #include <boost/filesystem.hpp>
  35  namespace fs_lib = boost::filesystem;
  36  #endif
  37  
  38  namespace i2p
  39  {
  40  namespace client
  41  {
  42  	// TODO: this is actually proxy class
  43  	class AddressBookFilesystemStorage: public AddressBookStorage
  44  	{
  45  		public:
  46  
  47  			AddressBookFilesystemStorage (): storage("addressbook", "b", "", "b32")
  48  			{
  49  				i2p::config::GetOption("persist.addressbook", m_IsPersist);
  50  				if (m_IsPersist)
  51  					i2p::config::GetOption("addressbook.hostsfile", m_HostsFile);
  52  			}
  53  			std::shared_ptr<const i2p::data::IdentityEx> GetAddress (const i2p::data::IdentHash& ident) override;
  54  			void AddAddress (std::shared_ptr<const i2p::data::IdentityEx> address) override;
  55  			void RemoveAddress (const i2p::data::IdentHash& ident) override;
  56  			void CleanUpCache () override;
  57  
  58  			bool Init () override;
  59  			int Load (Addresses& addresses) override;
  60  			int LoadLocal (Addresses& addresses) override;
  61  			int Save (const Addresses& addresses) override;
  62  
  63  			void SaveEtag (const i2p::data::IdentHash& subscription, const std::string& etag, const std::string& lastModified) override;
  64  			bool GetEtag (const i2p::data::IdentHash& subscription, std::string& etag, std::string& lastModified) override;
  65  			void ResetEtags () override;
  66  		
  67  		private:
  68  
  69  			int LoadFromFile (const std::string& filename, Addresses& addresses); // returns -1 if can't open file, otherwise number of records
  70  
  71  		private:
  72  
  73  			i2p::fs::HashedStorage storage;
  74  			std::string etagsPath, indexPath, localPath;
  75  			bool m_IsPersist;
  76  			std::string m_HostsFile; // file to dump hosts.txt, empty if not used
  77  			std::unordered_map<i2p::data::IdentHash, std::pair<std::vector<uint8_t>, uint64_t> > m_FullAddressCache; // ident hash -> (full ident buffer, last access timestamp) 
  78  			std::mutex m_FullAddressCacheMutex;
  79  	};
  80  
  81  	bool AddressBookFilesystemStorage::Init()
  82  	{
  83  		storage.SetPlace(i2p::fs::GetDataDir());
  84  		// init storage
  85  		if (storage.Init(i2p::data::GetBase32SubstitutionTable(), 32))
  86  		{
  87  			// init ETags
  88  			etagsPath = i2p::fs::StorageRootPath (storage, "etags");
  89  			if (!i2p::fs::Exists (etagsPath))
  90  				i2p::fs::CreateDirectory (etagsPath);
  91  			// init address files
  92  			indexPath = i2p::fs::StorageRootPath (storage, "addresses.csv");
  93  			localPath = i2p::fs::StorageRootPath (storage, "local.csv");
  94  			return true;
  95  		}
  96  		return false;
  97  	}
  98  
  99  	std::shared_ptr<const i2p::data::IdentityEx> AddressBookFilesystemStorage::GetAddress (const i2p::data::IdentHash& ident)
 100  	{
 101  		auto ts = i2p::util::GetMonotonicSeconds ();
 102  		{
 103  			std::lock_guard<std::mutex> l(m_FullAddressCacheMutex);
 104  			auto it = m_FullAddressCache.find (ident);
 105  			if (it != m_FullAddressCache.end ())
 106  			{
 107  				it->second.second = ts;
 108  				return std::make_shared<i2p::data::IdentityEx>(it->second.first.data (), it->second.first.size ());
 109  			}	
 110  		}
 111  	
 112  		if (!m_IsPersist)
 113  		{
 114  			LogPrint(eLogDebug, "Addressbook: Persistence is disabled");
 115  			return nullptr;
 116  		}
 117  		std::string filename = storage.Path(ident.ToBase32());
 118  		std::ifstream f(filename, std::ifstream::binary);
 119  		if (!f.is_open ()) 
 120  		{
 121  			LogPrint(eLogDebug, "Addressbook: Requested, but not found: ", filename);
 122  			return nullptr;
 123  		}
 124  
 125  		f.seekg (0,std::ios::end);
 126  		size_t len = f.tellg ();
 127  		if (len < i2p::data::DEFAULT_IDENTITY_SIZE) 
 128  		{
 129  			LogPrint (eLogError, "Addressbook: File ", filename, " is too short: ", len);
 130  			return nullptr;
 131  		}
 132  		f.seekg(0, std::ios::beg);
 133  		std::vector<uint8_t> buf(len);
 134  		f.read((char *)buf.data (), len);
 135  		if (!f)
 136  		{
 137  			LogPrint (eLogError, "Addressbook: Couldn't read ", filename);
 138  			return nullptr;
 139  		}	
 140  		{
 141  			std::lock_guard<std::mutex> l(m_FullAddressCacheMutex);
 142  			m_FullAddressCache.try_emplace (ident, buf, ts);
 143  		}	
 144  		return std::make_shared<i2p::data::IdentityEx>(buf.data (), len);
 145  	}
 146  
 147  	void AddressBookFilesystemStorage::AddAddress (std::shared_ptr<const i2p::data::IdentityEx> address)
 148  	{
 149  		if (!address) return;
 150  		size_t len = address->GetFullLen ();
 151  		std::vector<uint8_t> buf;
 152  		if (!len) return; // invalid address
 153  		{
 154  			std::lock_guard<std::mutex> l(m_FullAddressCacheMutex);
 155  			auto [it, inserted] = m_FullAddressCache.try_emplace (address->GetIdentHash(), len, i2p::util::GetMonotonicSeconds ());
 156  			if (inserted)
 157  				address->ToBuffer (it->second.first.data (), len);
 158  			if (m_IsPersist)
 159  				buf = it->second.first;
 160  		}
 161  		if (m_IsPersist && !buf.empty ())
 162  		{
 163  			std::string path = storage.Path(address->GetIdentHash().ToBase32());
 164  			std::ofstream f (path, std::ofstream::binary | std::ofstream::out);
 165  			if (!f.is_open ())	
 166  			{
 167  				LogPrint (eLogError, "Addressbook: Can't open file ", path);
 168  				return;
 169  			}
 170  			f.write ((const char *)buf.data (), len);
 171  		}	
 172  	}
 173  
 174  	void AddressBookFilesystemStorage::RemoveAddress (const i2p::data::IdentHash& ident)
 175  	{
 176  		{
 177  			std::lock_guard<std::mutex> l(m_FullAddressCacheMutex);
 178  			m_FullAddressCache.erase (ident);
 179  		}
 180  		if (!m_IsPersist) return;
 181  		storage.Remove( ident.ToBase32() );
 182  	}
 183  
 184  	int AddressBookFilesystemStorage::LoadFromFile (const std::string& filename, Addresses& addresses)
 185  	{
 186  		int num = 0;
 187  		std::ifstream f (filename, std::ifstream::in); // in text mode
 188  		if (!f) return -1;
 189  
 190  		addresses.clear ();
 191  		while (!f.eof ())
 192  		{
 193  			std::string s;
 194  			getline(f, s);
 195  			if (!s.length()) continue; // skip empty line
 196  
 197  			std::size_t pos = s.find(',');
 198  			if (pos != std::string::npos)
 199  			{
 200  				std::string name = s.substr(0, pos++);
 201  				std::string addr = s.substr(pos);
 202  
 203  				addresses[name] = std::make_shared<Address>(addr);
 204  				num++;
 205  			}
 206  		}
 207  		return num;
 208  	}
 209  
 210  	int AddressBookFilesystemStorage::Load (Addresses& addresses)
 211  	{
 212  		int num = LoadFromFile (indexPath, addresses);
 213  		if (num < 0)
 214  		{
 215  			LogPrint(eLogWarning, "Addressbook: Can't open ", indexPath);
 216  			return 0;
 217  		}
 218  		LogPrint(eLogInfo, "Addressbook: Using index file ", indexPath);
 219  		LogPrint (eLogInfo, "Addressbook: ", num, " addresses loaded from storage");
 220  
 221  		return num;
 222  	}
 223  
 224  	int AddressBookFilesystemStorage::LoadLocal (Addresses& addresses)
 225  	{
 226  		int num = LoadFromFile (localPath, addresses);
 227  		if (num < 0) return 0;
 228  		LogPrint (eLogInfo, "Addressbook: ", num, " local addresses loaded");
 229  		return num;
 230  	}
 231  
 232  	int AddressBookFilesystemStorage::Save (const Addresses& addresses)
 233  	{
 234  		if (addresses.empty())
 235  		{
 236  			LogPrint(eLogWarning, "Addressbook: Not saving empty addressbook");
 237  			return 0;
 238  		}
 239  
 240  		int num = 0;
 241  		{
 242  			// save index file
 243  			std::ofstream f (indexPath, std::ofstream::out); // in text mode
 244  			if (f.is_open ())
 245  			{
 246  				for (const auto& it: addresses)
 247  				{
 248  					if (it.second->IsValid ())
 249  					{
 250  						f << it.first << ",";
 251  						if (it.second->IsIdentHash ())
 252  							f << it.second->identHash.ToBase32 ();
 253  						else
 254  							f << it.second->blindedPublicKey->ToB33 ();
 255  						f << std::endl;
 256  						num++;
 257  					}
 258  					else
 259  						LogPrint (eLogWarning, "Addressbook: Invalid address ", it.first);
 260  				}
 261  				LogPrint (eLogInfo, "Addressbook: ", num, " addresses saved");
 262  			}
 263  			else
 264  				LogPrint (eLogWarning, "Addressbook: Can't open ", indexPath);
 265  		}
 266  		if (!m_HostsFile.empty ())
 267  		{
 268  			// dump full hosts.txt
 269  			std::ofstream f (m_HostsFile, std::ofstream::out); // in text mode
 270  			if (f.is_open ())
 271  			{
 272  				for (const auto& it: addresses)
 273  				{
 274  					std::shared_ptr<const i2p::data::IdentityEx> addr;
 275  					if (it.second->IsIdentHash ())
 276  					{
 277  						addr = GetAddress (it.second->identHash);
 278  						if (addr)
 279  							f << it.first << "=" << addr->ToBase64 () << std::endl;
 280  					}
 281  				}
 282  			}
 283  			else
 284  				LogPrint (eLogWarning, "Addressbook: Can't open ", m_HostsFile);
 285  		}
 286  
 287  		return num;
 288  	}
 289  
 290  	void AddressBookFilesystemStorage::SaveEtag (const i2p::data::IdentHash& subscription, const std::string& etag, const std::string& lastModified)
 291  	{
 292  		std::string fname = etagsPath + i2p::fs::dirSep + subscription.ToBase32 () + ".txt";
 293  		std::ofstream f (fname, std::ofstream::out | std::ofstream::trunc);
 294  		if (f)
 295  		{
 296  			f << etag << std::endl;
 297  			f<< lastModified << std::endl;
 298  		}
 299  	}
 300  
 301  	bool AddressBookFilesystemStorage::GetEtag (const i2p::data::IdentHash& subscription, std::string& etag, std::string& lastModified)
 302  	{
 303  		std::string fname = etagsPath + i2p::fs::dirSep + subscription.ToBase32 () + ".txt";
 304  		std::ifstream f (fname, std::ofstream::in);
 305  		if (!f || f.eof ()) return false;
 306  		std::getline (f, etag);
 307  		if (f.eof ()) return false;
 308  		std::getline (f, lastModified);
 309  		return true;
 310  	}
 311  
 312  	void AddressBookFilesystemStorage::ResetEtags ()
 313  	{
 314  		LogPrint (eLogError, "Addressbook: Resetting eTags");
 315  		for (fs_lib::directory_iterator it (etagsPath); it != fs_lib::directory_iterator (); ++it)
 316  		{
 317  			if (!fs_lib::is_regular_file (it->status ()))
 318  				continue;
 319  			fs_lib::remove (it->path ());
 320  		}
 321  	}
 322  
 323  	void AddressBookFilesystemStorage::CleanUpCache ()
 324  	{
 325  		auto ts = i2p::util::GetMonotonicSeconds ();
 326  		std::lock_guard<std::mutex> l(m_FullAddressCacheMutex);
 327  		for (auto it = m_FullAddressCache.begin (); it != m_FullAddressCache.end ();)
 328  		{
 329  			if (ts > it->second.second + ADDRESS_CACHE_EXPIRATION_TIMEOUT)
 330  				it = m_FullAddressCache.erase (it);
 331  			else
 332  				it++;
 333  		}	
 334  	}	
 335  	
 336  //---------------------------------------------------------------------
 337  
 338  	Address::Address (std::string_view b32):
 339  		addressType (eAddressInvalid)
 340  	{
 341  		if (b32.length () <= B33_ADDRESS_THRESHOLD)
 342  		{
 343  			if (identHash.FromBase32 (b32) > 0)
 344  				addressType = eAddressIndentHash;
 345  		}
 346  		else
 347  		{
 348  			blindedPublicKey = std::make_shared<i2p::data::BlindedPublicKey>(b32);
 349  			if (blindedPublicKey->IsValid ())
 350  				addressType = eAddressBlindedPublicKey;
 351  		}
 352  	}
 353  
 354  	Address::Address (const i2p::data::IdentHash& hash)
 355  	{
 356  		addressType = eAddressIndentHash;
 357  		identHash = hash;
 358  	}
 359  
 360  	AddressBook::AddressBook (): m_Storage(nullptr), m_IsLoaded (false),
 361  		m_NumRetries (0), m_DefaultSubscription (nullptr), m_SubscriptionsUpdateTimer (nullptr),
 362  		m_IsEnabled (true)
 363  	{
 364  	}
 365  
 366  	AddressBook::~AddressBook ()
 367  	{
 368  		Stop ();
 369  	}
 370  
 371  	void AddressBook::Start ()
 372  	{
 373  		i2p::config::GetOption("addressbook.enabled", m_IsEnabled);
 374  		if (m_IsEnabled)
 375  		{
 376  			if (!m_Storage)
 377  				m_Storage = new AddressBookFilesystemStorage;
 378  			m_Storage->Init();
 379  			LoadHosts (); /* try storage, then hosts.txt, then download */
 380  			StartSubscriptions ();
 381  			StartLookups ();
 382  			ScheduleCacheUpdate ();
 383  		}
 384  	}
 385  
 386  	void AddressBook::StartResolvers ()
 387  	{
 388  		LoadLocal ();
 389  	}
 390  
 391  	void AddressBook::Stop ()
 392  	{
 393  		StopLookups ();
 394  		StopSubscriptions ();
 395  		if (m_SubscriptionsUpdateTimer)
 396  		{
 397  			m_SubscriptionsUpdateTimer->cancel ();
 398  			m_SubscriptionsUpdateTimer = nullptr;
 399  		}
 400  		if (m_AddressCacheUpdateTimer)
 401  		{
 402  			m_AddressCacheUpdateTimer->cancel ();
 403  			m_AddressCacheUpdateTimer = nullptr;
 404  		}	
 405  		bool isDownloading = m_Downloading.valid ();
 406  		if (isDownloading)
 407  		{
 408  			if (m_Downloading.wait_for(std::chrono::seconds(0)) == std::future_status::ready)
 409  				isDownloading = false;
 410  			else	
 411  			{	
 412  				LogPrint (eLogInfo, "Addressbook: Subscriptions are downloading, abort");
 413  				for (int i = 0; i < 30; i++)
 414  				{
 415  					if (m_Downloading.wait_for(std::chrono::seconds(1)) == std::future_status::ready) // wait for 1 seconds
 416  					{
 417  						isDownloading = false;
 418  						LogPrint (eLogInfo, "Addressbook: Subscriptions download complete");
 419  						break;
 420  					}
 421  				}
 422  			}	
 423  			if (!isDownloading)
 424  				m_Downloading.get ();
 425  			else
 426  				LogPrint (eLogError, "Addressbook: Subscription download timeout");
 427  		}
 428  		if (m_Storage)
 429  		{
 430  			m_Storage->Save (m_Addresses);
 431  			delete m_Storage;
 432  			m_Storage = nullptr;
 433  		}
 434  		m_DefaultSubscription = nullptr;
 435  		m_Subscriptions.clear ();
 436  	}
 437  
 438  	std::shared_ptr<const Address> AddressBook::GetAddress (std::string_view address)
 439  	{
 440  		auto pos = address.find(".b32.i2p");
 441  		if (pos != std::string::npos)
 442  		{
 443  			auto addr = std::make_shared<const Address>(address.substr (0, pos));
 444  			return addr->IsValid () ? addr : nullptr;
 445  		}
 446  		else 
 447  #if __cplusplus >= 202002L // C++20
 448  		if (address.ends_with (".i2p"))
 449  #else
 450  		if (address.find (".i2p") != std::string::npos)
 451  #endif			
 452  		{
 453  			if (!m_IsEnabled) return nullptr;
 454  			auto addr = FindAddress (address);
 455  			if (!addr)
 456  				LookupAddress (address); // TODO:
 457  			return addr;
 458  		}
 459  		// if not .b32 we assume full base64 address
 460  		i2p::data::IdentityEx dest;
 461  		if (!dest.FromBase64 (address))
 462  			return nullptr;
 463  		return std::make_shared<const Address>(dest.GetIdentHash ());
 464  	}
 465  
 466  	std::shared_ptr<const Address> AddressBook::FindAddress (std::string_view address)
 467  	{
 468  		auto it = m_Addresses.find (address);
 469  		if (it != m_Addresses.end ())
 470  			return it->second;
 471  		return nullptr;
 472  	}
 473  
 474  	bool AddressBook::RecordExists (const std::string& address, const std::string& jump)
 475  	{
 476  		auto addr = FindAddress(address);
 477  		if (!addr)
 478  			return false;
 479  
 480  		auto pos = jump.find(".b32.i2p");
 481  		if (pos != std::string::npos)
 482  		{
 483  			i2p::data::IdentHash identHash;
 484  			if (identHash.FromBase32(jump.substr (0, pos)) && identHash == addr->identHash)
 485  				return true;
 486  		}	
 487  		else
 488  		{	
 489  			i2p::data::IdentityEx ident;
 490  			if (ident.FromBase64 (jump) && ident.GetIdentHash () == addr->identHash)
 491  				return true;
 492  		}
 493  			
 494  		return false;
 495  	}
 496  
 497  	void AddressBook::InsertAddress (const std::string& address, const std::string& jump)
 498  	{
 499  		auto pos = jump.find(".b32.i2p");
 500  		if (pos != std::string::npos)
 501  		{
 502  			m_Addresses[address] = std::make_shared<Address>(jump.substr (0, pos));
 503  			LogPrint (eLogInfo, "Addressbook: Added ", address," -> ", jump);
 504  		}
 505  		else
 506  		{
 507  			// assume base64
 508  			auto ident = std::make_shared<i2p::data::IdentityEx>();
 509  			if (ident->FromBase64 (jump))
 510  			{
 511  				if (m_Storage) m_Storage->AddAddress (ident);
 512  				m_Addresses[address] = std::make_shared<Address>(ident->GetIdentHash ());
 513  				LogPrint (eLogInfo, "Addressbook: Added ", address," -> ", ToAddress(ident->GetIdentHash ()));
 514  			}
 515  			else
 516  				LogPrint (eLogError, "Addressbook: Malformed address ", jump);
 517  		}
 518  	}
 519  
 520  	void AddressBook::InsertFullAddress (std::shared_ptr<const i2p::data::IdentityEx> address)
 521  	{
 522  		if (m_Storage) m_Storage->AddAddress (address);
 523  	}
 524  
 525  	std::shared_ptr<const i2p::data::IdentityEx> AddressBook::GetFullAddress (const std::string& address)
 526  	{
 527  		auto addr = GetAddress (address);
 528  		if (!addr || !addr->IsIdentHash ()) return nullptr;
 529  		return m_Storage ? m_Storage->GetAddress (addr->identHash) : nullptr;
 530  	}
 531  
 532  	void AddressBook::LoadHosts ()
 533  	{
 534  		if (!m_Storage) return;
 535  		if (m_Storage->Load (m_Addresses) > 0)
 536  		{
 537  			m_IsLoaded = true;
 538  			return;
 539  		}
 540  
 541  		// then try hosts.txt
 542  		std::ifstream f (i2p::fs::DataDirPath("hosts.txt"), std::ifstream::in); // in text mode
 543  		if (f.is_open ())
 544  		{
 545  			LoadHostsFromStream (f, false);
 546  			m_IsLoaded = true;
 547  		}
 548  
 549  		// reset eTags, because we don’t know how old hosts.txt is or can't load addressbook
 550  		m_Storage->ResetEtags ();
 551  	}
 552  
 553  	bool AddressBook::LoadHostsFromStream (std::istream& f, bool is_update)
 554  	{
 555  		std::unique_lock<std::mutex> l(m_AddressBookMutex);
 556  		int numAddresses = 0;
 557  		bool incomplete = false;
 558  		std::string s;
 559  		while (!f.eof ())
 560  		{
 561  			getline(f, s);
 562  
 563  			if (!s.length() || s[0] == '#')
 564  				continue; // skip empty or comment line
 565  
 566  			size_t pos = s.find('=');
 567  
 568  			if (pos != std::string::npos)
 569  			{
 570  				std::string_view name = std::string_view(s).substr(0, pos++);
 571  				std::string_view addr = std::string_view(s).substr(pos);
 572  
 573  				size_t pos = addr.find('#');
 574  				if (pos != addr.npos)
 575  					addr = addr.substr(0, pos); // remove comments
 576  #if __cplusplus >= 202002L // C++20
 577  				if (name.ends_with (".b32.i2p"))
 578  #else					
 579  				if (name.find(".b32.i2p") != name.npos)
 580  #endif					
 581  				{
 582  					LogPrint (eLogError, "Addressbook: Skipped adding of b32 address: ", name);
 583  					continue;
 584  				}
 585  
 586  #if __cplusplus >= 202002L // C++20
 587  				if (!name.ends_with (".i2p"))
 588  #else	
 589  				if (name.find(".i2p") == name.npos)
 590  #endif					
 591  				{
 592  					LogPrint (eLogError, "Addressbook: Malformed domain: ", name);
 593  					continue;
 594  				}
 595  
 596  				auto ident = std::make_shared<i2p::data::IdentityEx> ();
 597  				if (!ident->FromBase64(addr)) 
 598  				{
 599  					LogPrint (eLogError, "Addressbook: Malformed address ", addr, " for ", name);
 600  					incomplete = f.eof ();
 601  					continue;
 602  				}
 603  				numAddresses++;
 604  				auto it = m_Addresses.find (name);
 605  				if (it != m_Addresses.end ()) // already exists ?
 606  				{
 607  					if (it->second->IsIdentHash () && it->second->identHash != ident->GetIdentHash () && // address changed?
 608  						ident->GetSigningKeyType () != i2p::data::SIGNING_KEY_TYPE_DSA_SHA1) // don't replace by DSA
 609  					{
 610  						it->second->identHash = ident->GetIdentHash ();
 611  						if (m_Storage)
 612  						{	
 613  							m_Storage->AddAddress (ident);
 614  							m_Storage->RemoveAddress (it->second->identHash);
 615  						}	
 616  						LogPrint (eLogInfo, "Addressbook: Updated host: ", name);
 617  					}
 618  				}
 619  				else
 620  				{
 621  					m_Addresses.emplace (name, std::make_shared<Address>(ident->GetIdentHash ()));
 622  					if (m_Storage) m_Storage->AddAddress (ident);
 623  					if (is_update)
 624  						LogPrint (eLogInfo, "Addressbook: Added new host: ", name);
 625  				}
 626  			}
 627  			else
 628  				incomplete = f.eof ();
 629  		}
 630  		LogPrint (eLogInfo, "Addressbook: ", numAddresses, " addresses processed");
 631  		if (numAddresses > 0)
 632  		{
 633  			if (!incomplete) m_IsLoaded = true;
 634  			if (m_Storage) m_Storage->Save (m_Addresses);
 635  		}
 636  		return !incomplete;
 637  	}
 638  
 639  	void AddressBook::LoadSubscriptions ()
 640  	{
 641  		if (!m_Subscriptions.size ())
 642  		{
 643  			std::ifstream f (i2p::fs::DataDirPath ("subscriptions.txt"), std::ifstream::in); // in text mode
 644  			if (f.is_open ())
 645  			{
 646  				std::string s;
 647  				while (!f.eof ())
 648  				{
 649  					getline(f, s);
 650  					if (s.empty () || s[0] == '#') continue; // skip empty line or comment
 651  					m_Subscriptions.push_back (std::make_shared<AddressBookSubscription> (*this, s));
 652  				}
 653  				LogPrint (eLogInfo, "Addressbook: ", m_Subscriptions.size (), " subscriptions urls loaded");
 654  				LogPrint (eLogWarning, "Addressbook: subscriptions.txt usage is deprecated, use config file instead");
 655  			}
 656  			else
 657  			{
 658  				LogPrint (eLogInfo, "Addressbook: Loading subscriptions from config");
 659  				// using config file items
 660  				std::string subscriptionURLs; i2p::config::GetOption("addressbook.subscriptions", subscriptionURLs);
 661  				std::vector<std::string> subsList;
 662  				boost::split(subsList, subscriptionURLs, boost::is_any_of(","), boost::token_compress_on);
 663  
 664  				for (const auto& s: subsList)
 665  					if (!s.empty ())
 666  						m_Subscriptions.push_back (std::make_shared<AddressBookSubscription> (*this, s));
 667  				LogPrint (eLogInfo, "Addressbook: ", m_Subscriptions.size (), " subscriptions urls loaded");
 668  			}
 669  		}
 670  		else
 671  			LogPrint (eLogError, "Addressbook: Subscriptions already loaded");
 672  	}
 673  
 674  	void AddressBook::LoadLocal ()
 675  	{
 676  		if (!m_Storage) return;
 677  		AddressBookStorage::Addresses localAddresses;
 678  		m_Storage->LoadLocal (localAddresses);
 679  		for (const auto& it: localAddresses)
 680  		{
 681  			if (!it.second->IsIdentHash ()) continue; // skip blinded for now
 682  			auto dot = it.first.find ('.');
 683  			if (dot != std::string::npos)
 684  			{
 685  				auto domain = it.first.substr (dot + 1);
 686  				auto it1 = m_Addresses.find (domain); // find domain in our addressbook
 687  				if (it1 != m_Addresses.end () && it1->second->IsIdentHash ())
 688  				{
 689  					auto dest = context.FindLocalDestination (it1->second->identHash);
 690  					if (dest)
 691  					{
 692  						// address is ours
 693  						std::shared_ptr<AddressResolver> resolver;
 694  						auto it2 = m_Resolvers.find (it1->second->identHash);
 695  						if (it2 != m_Resolvers.end ())
 696  							resolver = it2->second; // resolver exists
 697  						else
 698  						{
 699  							// create new resolver
 700  							resolver = std::make_shared<AddressResolver>(dest);
 701  							m_Resolvers.insert (std::make_pair(it1->second->identHash, resolver));
 702  						}
 703  						resolver->AddAddress (it.first, it.second->identHash);
 704  					}
 705  				}
 706  			}
 707  		}
 708  	}
 709  
 710  	bool AddressBook::GetEtag (const i2p::data::IdentHash& subscription, std::string& etag, std::string& lastModified)
 711  	{
 712  		if (m_Storage)
 713  			return m_Storage->GetEtag (subscription, etag, lastModified);
 714  		else
 715  			return false;
 716  	}
 717  
 718  	void AddressBook::DownloadComplete (bool success, const i2p::data::IdentHash& subscription, const std::string& etag, const std::string& lastModified)
 719  	{
 720  		m_NumRetries++;
 721  		int nextUpdateTimeout = m_NumRetries*CONTINIOUS_SUBSCRIPTION_RETRY_TIMEOUT;
 722  		if (m_NumRetries > CONTINIOUS_SUBSCRIPTION_MAX_NUM_RETRIES || nextUpdateTimeout > CONTINIOUS_SUBSCRIPTION_UPDATE_TIMEOUT)
 723  			nextUpdateTimeout = CONTINIOUS_SUBSCRIPTION_UPDATE_TIMEOUT;
 724  		if (success)
 725  		{
 726  			m_NumRetries = 0;
 727  			if (m_DefaultSubscription) m_DefaultSubscription = nullptr;
 728  			if (m_IsLoaded)
 729  				nextUpdateTimeout = CONTINIOUS_SUBSCRIPTION_UPDATE_TIMEOUT;
 730  			else
 731  				m_IsLoaded = true;
 732  			if (m_Storage) m_Storage->SaveEtag (subscription, etag, lastModified);
 733  		}
 734  		if (m_SubscriptionsUpdateTimer)
 735  		{
 736  			m_SubscriptionsUpdateTimer->expires_from_now (boost::posix_time::minutes(nextUpdateTimeout));
 737  			m_SubscriptionsUpdateTimer->async_wait (std::bind (&AddressBook::HandleSubscriptionsUpdateTimer,
 738  				this, std::placeholders::_1));
 739  		}
 740  	}
 741  
 742  	void AddressBook::StartSubscriptions ()
 743  	{
 744  		LoadSubscriptions ();
 745  		if (m_IsLoaded && m_Subscriptions.empty ()) return;
 746  
 747  		auto dest = i2p::client::context.GetSharedLocalDestination ();
 748  		if (dest)
 749  		{
 750  			m_SubscriptionsUpdateTimer = std::make_unique<boost::asio::deadline_timer>(dest->GetService ());
 751  			m_SubscriptionsUpdateTimer->expires_from_now (boost::posix_time::minutes(INITIAL_SUBSCRIPTION_UPDATE_TIMEOUT));
 752  			m_SubscriptionsUpdateTimer->async_wait (std::bind (&AddressBook::HandleSubscriptionsUpdateTimer,
 753  				this, std::placeholders::_1));
 754  		}
 755  		else
 756  			LogPrint (eLogCritical, "Addressbook: Can't start subscriptions: missing shared local destination");
 757  	}
 758  
 759  	void AddressBook::StopSubscriptions ()
 760  	{
 761  		if (m_SubscriptionsUpdateTimer)
 762  			m_SubscriptionsUpdateTimer->cancel ();
 763  	}
 764  
 765  	void AddressBook::HandleSubscriptionsUpdateTimer (const boost::system::error_code& ecode)
 766  	{
 767  		if (ecode != boost::asio::error::operation_aborted)
 768  		{
 769  			auto dest = i2p::client::context.GetSharedLocalDestination ();
 770  			if (!dest) {
 771  				LogPrint(eLogWarning, "Addressbook: Missing local destination, skip subscription update");
 772  				return;
 773  			}
 774  			bool isDownloading = m_Downloading.valid ();
 775  			if (isDownloading && m_Downloading.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active?
 776  			{
 777  				m_Downloading.get ();
 778  				isDownloading = false;
 779  			}	
 780  			if (!isDownloading && dest->IsReady ())
 781  			{
 782  				if (!m_IsLoaded)
 783  				{
 784  					// download it from default subscription
 785  					LogPrint (eLogInfo, "Addressbook: Trying to download it from default subscription.");
 786  					std::string defaultSubURL; i2p::config::GetOption("addressbook.defaulturl", defaultSubURL);
 787  					if (!m_DefaultSubscription)
 788  						m_DefaultSubscription = std::make_shared<AddressBookSubscription>(*this, defaultSubURL);
 789  					m_Downloading = std::async (std::launch::async,
 790  						std::bind (&AddressBookSubscription::CheckUpdates, m_DefaultSubscription));
 791  				}
 792  				else if (!m_Subscriptions.empty ())
 793  				{
 794  					// pick random subscription
 795  					auto ind = rand () % m_Subscriptions.size();
 796  					m_Downloading = std::async (std::launch::async,
 797  						std::bind (&AddressBookSubscription::CheckUpdates, m_Subscriptions[ind]));
 798  				}
 799  			}
 800  			else
 801  			{
 802  				// try it again later
 803  				m_SubscriptionsUpdateTimer->expires_from_now (boost::posix_time::minutes(INITIAL_SUBSCRIPTION_RETRY_TIMEOUT));
 804  				m_SubscriptionsUpdateTimer->async_wait (std::bind (&AddressBook::HandleSubscriptionsUpdateTimer,
 805  					this, std::placeholders::_1));
 806  			}
 807  		}
 808  	}
 809  
 810  	void AddressBook::StartLookups ()
 811  	{
 812  		auto dest = i2p::client::context.GetSharedLocalDestination ();
 813  		if (dest)
 814  		{
 815  			auto datagram = dest->GetDatagramDestination ();
 816  			if (!datagram)
 817  				datagram = dest->CreateDatagramDestination ();
 818  			datagram->SetReceiver (std::bind (&AddressBook::HandleLookupResponse, this,
 819  				std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4, std::placeholders::_5),
 820  				ADDRESS_RESPONSE_DATAGRAM_PORT);
 821  		}
 822  	}
 823  
 824  	void AddressBook::StopLookups ()
 825  	{
 826  		auto dest = i2p::client::context.GetSharedLocalDestination ();
 827  		if (dest)
 828  		{
 829  			auto datagram = dest->GetDatagramDestination ();
 830  			if (datagram) datagram->ResetReceiver (ADDRESS_RESPONSE_DATAGRAM_PORT);
 831  		}
 832  	}
 833  
 834  	void AddressBook::LookupAddress (std::string_view address)
 835  	{
 836  		std::shared_ptr<const Address> addr;
 837  		auto dot = address.find ('.');
 838  		if (dot != std::string::npos)
 839  			addr = FindAddress (address.substr (dot + 1));
 840  		if (!addr || !addr->IsIdentHash ()) // TODO:
 841  		{
 842  			LogPrint (eLogError, "Addressbook: Can't find domain for ", address);
 843  			return;
 844  		}
 845  
 846  		auto dest = i2p::client::context.GetSharedLocalDestination ();
 847  		if (dest)
 848  		{
 849  			auto datagram = dest->GetDatagramDestination ();
 850  			if (datagram)
 851  			{
 852  				uint32_t nonce;
 853  				RAND_bytes ((uint8_t *)&nonce, 4);
 854  				{
 855  					std::unique_lock<std::mutex> l(m_LookupsMutex);
 856  					m_Lookups[nonce] = address;
 857  				}
 858  				LogPrint (eLogDebug, "Addressbook: Lookup of ", address, " to ", addr->identHash.ToBase32 (), " nonce=", nonce);
 859  				size_t len = address.length () + 9;
 860  				uint8_t * buf = new uint8_t[len];
 861  				memset (buf, 0, 4);
 862  				htobe32buf (buf + 4, nonce);
 863  				buf[8] = address.length ();
 864  				memcpy (buf + 9, address.data (), address.length ());
 865  				datagram->SendDatagramTo (buf, len, addr->identHash, ADDRESS_RESPONSE_DATAGRAM_PORT, ADDRESS_RESOLVER_DATAGRAM_PORT);
 866  				delete[] buf;
 867  			}
 868  		}
 869  	}
 870  
 871  	void AddressBook::HandleLookupResponse (const i2p::data::IdentityEx& from, uint16_t fromPort, uint16_t toPort, const uint8_t * buf, size_t len)
 872  	{
 873  		if (len < 44)
 874  		{
 875  			LogPrint (eLogError, "Addressbook: Lookup response is too short ", len);
 876  			return;
 877  		}
 878  		uint32_t nonce = bufbe32toh (buf + 4);
 879  		LogPrint (eLogDebug, "Addressbook: Lookup response received from ", from.GetIdentHash ().ToBase32 (), " nonce=", nonce);
 880  		std::string address;
 881  		{
 882  			std::unique_lock<std::mutex> l(m_LookupsMutex);
 883  			auto it = m_Lookups.find (nonce);
 884  			if (it != m_Lookups.end ())
 885  			{
 886  				address = it->second;
 887  				m_Lookups.erase (it);
 888  			}
 889  		}
 890  		if (address.length () > 0)
 891  		{
 892  			// TODO: verify from
 893  			i2p::data::IdentHash hash(buf + 8);
 894  			if (!hash.IsZero ())
 895  				m_Addresses[address] = std::make_shared<Address>(hash);
 896  			else
 897  				LogPrint (eLogInfo, "AddressBook: Lookup response: ", address, " not found");
 898  		}
 899  	}
 900  
 901  	void AddressBook::ScheduleCacheUpdate ()
 902  	{
 903  		if (!m_AddressCacheUpdateTimer)
 904  		{
 905  			auto dest = i2p::client::context.GetSharedLocalDestination ();
 906  			if(dest)
 907  				m_AddressCacheUpdateTimer = std::make_unique<boost::asio::deadline_timer>(dest->GetService ());
 908  		}	
 909  		if (m_AddressCacheUpdateTimer)
 910  		{	
 911  			m_AddressCacheUpdateTimer->expires_from_now (boost::posix_time::seconds(ADDRESS_CACHE_UPDATE_INTERVAL ));
 912  			m_AddressCacheUpdateTimer->async_wait (
 913  				[this](const boost::system::error_code& ecode) 
 914  				{
 915  					if (ecode != boost::asio::error::operation_aborted)
 916  					{
 917  						if (m_Storage) m_Storage->CleanUpCache ();
 918  						ScheduleCacheUpdate ();
 919  					}	
 920  				});
 921  		}	
 922  	}	
 923  		
 924  	AddressBookSubscription::AddressBookSubscription (AddressBook& book, std::string_view link):
 925  		m_Book (book), m_Link (link)
 926  	{
 927  	}
 928  
 929  	void AddressBookSubscription::CheckUpdates ()
 930  	{
 931  		i2p::util::SetThreadName("Addressbook");
 932  
 933  		bool result = MakeRequest ();
 934  		m_Book.DownloadComplete (result, m_Ident, m_Etag, m_LastModified);
 935  	}
 936  
 937  	bool AddressBookSubscription::MakeRequest ()
 938  	{
 939  		i2p::http::URL url;
 940  		// must be run in separate thread
 941  		LogPrint (eLogInfo, "Addressbook: Downloading hosts database from ", m_Link);
 942  		if (!url.parse(m_Link))
 943  		{
 944  			LogPrint(eLogError, "Addressbook: Failed to parse url: ", m_Link);
 945  			return false;
 946  		}
 947  		auto addr = m_Book.GetAddress (url.host);
 948  		if (!addr || !addr->IsIdentHash ())
 949  		{
 950  			LogPrint (eLogError, "Addressbook: Can't resolve ", url.host);
 951  			return false;
 952  		}
 953  		else
 954  			m_Ident = addr->identHash;
 955  		// save url parts for later use
 956  		std::string dest_host = url.host;
 957  		int         dest_port = url.port ? url.port : 80;
 958  		// try to create stream to addressbook site
 959  		auto stream = i2p::client::context.GetSharedLocalDestination ()->CreateStream (m_Ident, dest_port);
 960  		if (!stream)
 961  		{
 962  			LogPrint (eLogError, "Addressbook: LeaseSet for address ", url.host, " not found");
 963  			return false;
 964  		}
 965  		if (m_Etag.empty() && m_LastModified.empty())
 966  		{
 967  			m_Book.GetEtag (m_Ident, m_Etag, m_LastModified);
 968  			LogPrint (eLogDebug, "Addressbook: Loaded for ", url.host, ": ETag: ", m_Etag, ", Last-Modified: ", m_LastModified);
 969  		}
 970  		// create http request & send it
 971  		i2p::http::HTTPReq req;
 972  		req.AddHeader("Host", dest_host);
 973  		req.AddHeader("User-Agent", "Wget/1.11.4");
 974  		req.AddHeader("Accept-Encoding", "gzip");
 975  		req.AddHeader("X-Accept-Encoding", "x-i2p-gzip;q=1.0, identity;q=0.5, deflate;q=0, gzip;q=0, *;q=0");
 976  		req.AddHeader("Connection", "close");
 977  		if (!m_Etag.empty())
 978  			req.AddHeader("If-None-Match", m_Etag);
 979  		if (!m_LastModified.empty())
 980  			req.AddHeader("If-Modified-Since", m_LastModified);
 981  		// convert url to relative
 982  		url.schema  = "";
 983  		url.host    = "";
 984  		req.uri     = url.to_string();
 985  		req.version = "HTTP/1.1";
 986  		std::string request = req.to_string();
 987  		stream->Send ((const uint8_t *) request.data(), request.length());
 988  		// read response
 989  		std::string response;
 990  		uint8_t recv_buf[4096];
 991  		bool end = false;
 992  		int numAttempts = 0;
 993  		while (!end)
 994  		{
 995  			size_t received = stream->Receive (recv_buf, 4096, SUBSCRIPTION_REQUEST_TIMEOUT);
 996  			if (received)
 997  			{
 998  				response.append ((char *)recv_buf, received);
 999  				if (!stream->IsOpen ()) end = true;
1000  			}
1001  			else if (!stream->IsOpen ())
1002  				end = true;
1003  			else
1004  			{
1005  				LogPrint (eLogError, "Addressbook: Subscriptions request timeout expired");
1006  				numAttempts++;
1007  				if (numAttempts > 5) end = true;
1008  			}
1009  		}
1010  		// process remaining buffer
1011  		while (size_t len = stream->ReadSome (recv_buf, sizeof(recv_buf)))
1012  			response.append ((char *)recv_buf, len);
1013  		// parse response
1014  		i2p::http::HTTPRes res;
1015  		int res_head_len = res.parse(response);
1016  		if (res_head_len < 0)
1017  		{
1018  			LogPrint(eLogError, "Addressbook: Can't parse http response from ", dest_host);
1019  			return false;
1020  		}
1021  		if (res_head_len == 0)
1022  		{
1023  			LogPrint(eLogError, "Addressbook: Incomplete http response from ", dest_host, ", interrupted by timeout");
1024  			return false;
1025  		}
1026  		// assert: res_head_len > 0
1027  		response.erase(0, res_head_len);
1028  		if (res.code == 304)
1029  		{
1030  			LogPrint (eLogInfo, "Addressbook: No updates from ", dest_host, ", code 304");
1031  			return false;
1032  		}
1033  		if (res.code != 200)
1034  		{
1035  			LogPrint (eLogWarning, "Adressbook: Can't get updates from ", dest_host, ", response code ", res.code);
1036  			return false;
1037  		}
1038  		int len = res.content_length();
1039  		if (response.empty())
1040  		{
1041  			LogPrint(eLogError, "Addressbook: Empty response from ", dest_host, ", expected ", len, " bytes");
1042  			return false;
1043  		}
1044  		if (!res.is_gzipped () && len > 0 && len != (int) response.length())
1045  		{
1046  			LogPrint(eLogError, "Addressbook: Response size mismatch, expected: ", len, ", got: ", response.length(), "bytes");
1047  			return false;
1048  		}
1049  		// assert: res.code == 200
1050  		auto it = res.headers.find("ETag");
1051  		if (it != res.headers.end()) m_Etag = it->second;
1052  		it = res.headers.find("Last-Modified");
1053  		if (it != res.headers.end()) m_LastModified = it->second;
1054  		if (res.is_chunked())
1055  		{
1056  			std::stringstream in(response), out;
1057  			i2p::http::MergeChunkedResponse (in, out);
1058  			response = out.str();
1059  		}
1060  		if (res.is_gzipped())
1061  		{
1062  			std::stringstream out;
1063  			i2p::data::GzipInflator inflator;
1064  			inflator.Inflate ((const uint8_t *) response.data(), response.length(), out);
1065  			if (out.fail())
1066  			{
1067  				LogPrint(eLogError, "Addressbook: Can't gunzip http response");
1068  				return false;
1069  			}
1070  			response = out.str();
1071  		}
1072  		std::stringstream ss(response);
1073  		LogPrint (eLogInfo, "Addressbook: Got update from ", dest_host);
1074  		m_Book.LoadHostsFromStream (ss, true);
1075  		return true;
1076  	}
1077  
1078  	AddressResolver::AddressResolver (std::shared_ptr<ClientDestination> destination):
1079  		m_LocalDestination (destination)
1080  	{
1081  		if (m_LocalDestination)
1082  		{
1083  			auto datagram = m_LocalDestination->GetDatagramDestination ();
1084  			if (!datagram)
1085  				datagram = m_LocalDestination->CreateDatagramDestination ();
1086  			datagram->SetReceiver (std::bind (&AddressResolver::HandleRequest, this,
1087  				std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4, std::placeholders::_5),
1088  				ADDRESS_RESOLVER_DATAGRAM_PORT);
1089  		}
1090  	}
1091  
1092  	AddressResolver::~AddressResolver ()
1093  	{
1094  		if (m_LocalDestination)
1095  		{
1096  			auto datagram = m_LocalDestination->GetDatagramDestination ();
1097  			if (datagram)
1098  				datagram->ResetReceiver (ADDRESS_RESOLVER_DATAGRAM_PORT);
1099  		}
1100  	}
1101  
1102  	void AddressResolver::HandleRequest (const i2p::data::IdentityEx& from, uint16_t fromPort, uint16_t toPort, const uint8_t * buf, size_t len)
1103  	{
1104  		if (len < 9 || len < buf[8] + 9U)
1105  		{
1106  			LogPrint (eLogError, "Addressbook: Address request is too short ", len);
1107  			return;
1108  		}
1109  		// read requested address
1110  		uint8_t l = buf[8];
1111  		char address[255];
1112  		memcpy (address, buf + 9, l);
1113  		address[l] = 0;
1114  		LogPrint (eLogDebug, "Addressbook: Address request ", address);
1115  		// send response
1116  		uint8_t response[44];
1117  		memset (response, 0, 4); // reserved
1118  		memcpy (response + 4, buf + 4, 4); // nonce
1119  		auto it = m_LocalAddresses.find (address); // address lookup
1120  		if (it != m_LocalAddresses.end ())
1121  			memcpy (response + 8, it->second, 32); // ident
1122  		else
1123  			memset (response + 8, 0, 32); // not found
1124  		memset (response + 40, 0, 4); // set expiration time to zero
1125  		m_LocalDestination->GetDatagramDestination ()->SendDatagramTo (response, 44, from.GetIdentHash(), toPort, fromPort);
1126  	}
1127  
1128  	void AddressResolver::AddAddress (const std::string& name, const i2p::data::IdentHash& ident)
1129  	{
1130  		m_LocalAddresses[name] = ident;
1131  	}
1132  
1133  }
1134  }