1  /*
   2  * Copyright (c) 2013-2026, The PurpleI2P Project
   3  *
   4  * This file is part of Purple i2pd project and licensed under BSD3
   5  *
   6  * See full license text in LICENSE file at top of project tree
   7  */
   8  
   9  #include <string.h>
  10  #include <fstream>
  11  #include <vector>
  12  #include <map>
  13  #include <boost/asio.hpp>
  14  #include <stdexcept>
  15  
  16  #include "I2PEndian.h"
  17  #include "Base.h"
  18  #include "Crypto.h"
  19  #include "Log.h"
  20  #include "Timestamp.h"
  21  #include "I2NPProtocol.h"
  22  #include "Tunnel.h"
  23  #include "Transports.h"
  24  #include "NTCP2.h"
  25  #include "RouterContext.h"
  26  #include "Garlic.h"
  27  #include "ECIESX25519AEADRatchetSession.h"
  28  #include "Config.h"
  29  #include "NetDb.hpp"
  30  #include "util.h"
  31  
  32  using namespace i2p::transport;
  33  
  34  namespace i2p
  35  {
  36  namespace data
  37  {
  38  	NetDb netdb;
  39  
  40  	NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr),
  41  		m_Storage("netDb", "r", "routerInfo-", "dat"), m_PersistProfiles (true),
  42  		m_LastExploratorySelectionUpdateTime (0), m_Rng(i2p::util::GetMonotonicMicroseconds () % 1000000LL)
  43  	{
  44  	}
  45  
  46  	NetDb::~NetDb ()
  47  	{
  48  		Stop ();
  49  		delete m_Reseeder;
  50  	}
  51  
  52  	void NetDb::Start ()
  53  	{
  54  		m_Storage.SetPlace(i2p::fs::GetDataDir());
  55  		m_Storage.Init(i2p::data::GetBase64SubstitutionTable(), 64);
  56  		InitProfilesStorage ();
  57  		m_Families.LoadCertificates ();
  58  		Load ();
  59  
  60  		if (!m_Requests)
  61  		{
  62  			m_Requests = std::make_shared<NetDbRequests>();
  63  			m_Requests->Start ();
  64  		}
  65  
  66  		uint16_t threshold; i2p::config::GetOption("reseed.threshold", threshold);
  67  		if (m_RouterInfos.size () < threshold || m_Floodfills.GetSize () < NETDB_MIN_FLOODFILLS) // reseed if # of router less than threshold or too few floodfiils
  68  		{
  69  			Reseed ();
  70  		}
  71  		else if (!GetRandomRouter (i2p::context.GetSharedRouterInfo (), false, false, false))
  72  			Reseed (); // we don't have a router we can connect to. Trying to reseed
  73  
  74  		auto it = m_RouterInfos.find (i2p::context.GetIdentHash ());
  75  		if (it != m_RouterInfos.end ())
  76  		{
  77  			// remove own router
  78  			m_Floodfills.Remove (it->second->GetIdentHash ());
  79  			m_RouterInfos.erase (it);
  80  		}
  81  		// insert own router
  82  		m_RouterInfos.emplace (i2p::context.GetIdentHash (), i2p::context.GetSharedRouterInfo ());
  83  		if (i2p::context.IsFloodfill ())
  84  			m_Floodfills.Insert (i2p::context.GetSharedRouterInfo ());
  85  
  86  		i2p::config::GetOption("persist.profiles", m_PersistProfiles);
  87  
  88  		m_IsRunning = true;
  89  		m_Thread = new std::thread (std::bind (&NetDb::Run, this));
  90  	}
  91  
  92  	void NetDb::Stop ()
  93  	{
  94  		if (m_Requests)
  95  			m_Requests->Stop ();
  96  		if (m_IsRunning)
  97  		{
  98  			if (m_PersistProfiles)
  99  				SaveProfiles ();
 100  			DeleteObsoleteProfiles ();
 101  			m_RouterInfos.clear ();
 102  			m_Floodfills.Clear ();
 103  			if (m_Thread)
 104  			{
 105  				m_IsRunning = false;
 106  				m_Queue.WakeUp ();
 107  				m_Thread->join ();
 108  				delete m_Thread;
 109  				m_Thread = 0;
 110  			}
 111  			m_LeaseSets.clear();
 112  		}
 113  		m_Requests = nullptr;
 114  	}
 115  
 116  	void NetDb::Run ()
 117  	{
 118  		i2p::util::SetThreadName("NetDB");
 119  
 120  		uint64_t lastManage = 0;
 121  		uint64_t lastProfilesCleanup = i2p::util::GetMonotonicMilliseconds (),
 122  			lastObsoleteProfilesCleanup = lastProfilesCleanup, lastApplyingProfileUpdates = lastProfilesCleanup;
 123  		int16_t profilesCleanupVariance = 0, obsoleteProfilesCleanVariance = 0, applyingProfileUpdatesVariance = 0;
 124  
 125  		std::list<std::shared_ptr<const I2NPMessage> > msgs;
 126  		while (m_IsRunning)
 127  		{
 128  			try
 129  			{
 130  				if (m_Queue.Wait (1,0)) // 1 sec
 131  				{
 132  					m_Queue.GetWholeQueue (msgs);
 133  					while (!msgs.empty ())
 134  					{
 135  						auto msg = msgs.front (); msgs.pop_front ();
 136  						if (!msg) continue;
 137  						LogPrint(eLogDebug, "NetDb: Got request with type ", (int) msg->GetTypeID ());
 138  						switch (msg->GetTypeID ())
 139  						{
 140  							case eI2NPDatabaseStore:
 141  								HandleDatabaseStoreMsg (msg);
 142  							break;
 143  							case eI2NPDatabaseLookup:
 144  								HandleDatabaseLookupMsg (msg);
 145  							break;
 146  							default: // WTF?
 147  								LogPrint (eLogError, "NetDb: Unexpected message type ", (int) msg->GetTypeID ());
 148  								//i2p::HandleI2NPMessage (msg);
 149  						}
 150  					}
 151  				}
 152  				if (!m_IsRunning) break;
 153  				if (!i2p::transport::transports.IsOnline () || !i2p::transport::transports.IsRunning ())
 154  					continue; // don't manage netdb when offline or transports are not running
 155  
 156  				uint64_t mts = i2p::util::GetMonotonicMilliseconds ();
 157  				if (mts >= lastManage + 60000) // manage routers and leasesets every minute
 158  				{
 159  					if (lastManage)
 160  					{
 161  						ManageRouterInfos ();
 162  						ManageLeaseSets ();
 163  					}
 164  					lastManage = mts;
 165  				}
 166  
 167  				if (mts >= lastProfilesCleanup + (uint64_t)(i2p::data::PEER_PROFILE_AUTOCLEAN_TIMEOUT + profilesCleanupVariance)*1000)
 168  				{
 169  					m_RouterProfilesPool.CleanUpMt ();
 170  					if (m_PersistProfiles)
 171  					{
 172  						bool isSaving = m_SavingProfiles.valid ();
 173  						if (isSaving && m_SavingProfiles.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active?
 174  						{
 175  							m_SavingProfiles.get ();
 176  							isSaving = false;
 177  						}
 178  						if (!isSaving)
 179  							m_SavingProfiles = PersistProfiles ();
 180  						else
 181  							LogPrint (eLogWarning, "NetDb: Can't persist profiles. Profiles are being saved to disk");
 182  					}
 183  					lastProfilesCleanup = mts;
 184  					profilesCleanupVariance = m_Rng () % i2p::data::PEER_PROFILE_AUTOCLEAN_VARIANCE;
 185  				}
 186  
 187  				if (mts >= lastObsoleteProfilesCleanup + (uint64_t)(i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT + obsoleteProfilesCleanVariance)*1000)
 188  				{
 189  					bool isDeleting = m_DeletingProfiles.valid ();
 190  					if (isDeleting && m_DeletingProfiles.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active?
 191  					{
 192  						m_DeletingProfiles.get ();
 193  						isDeleting = false;
 194  					}
 195  					if (!isDeleting)
 196  						m_DeletingProfiles = DeleteObsoleteProfiles ();
 197  					else
 198  						LogPrint (eLogWarning, "NetDb: Can't delete profiles. Profiles are being deleted from disk");
 199  					lastObsoleteProfilesCleanup = mts;
 200  					obsoleteProfilesCleanVariance = m_Rng () % i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_VARIANCE;
 201  				}
 202  				if (mts >= lastApplyingProfileUpdates + i2p::data::PEER_PROFILE_APPLY_POSTPONED_TIMEOUT + applyingProfileUpdatesVariance)
 203  				{
 204  					bool isApplying = m_ApplyingProfileUpdates.valid ();
 205  					if (isApplying && m_ApplyingProfileUpdates.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active?
 206  					{
 207  						m_ApplyingProfileUpdates.get ();
 208  						isApplying = false;
 209  					}
 210  					if (!isApplying)
 211  						m_ApplyingProfileUpdates = i2p::data::FlushPostponedRouterProfileUpdates ();
 212  					lastApplyingProfileUpdates = mts;
 213  					applyingProfileUpdatesVariance = m_Rng () % i2p::data::PEER_PROFILE_APPLY_POSTPONED_TIMEOUT_VARIANCE;
 214  				}
 215  			}
 216  			catch (std::exception& ex)
 217  			{
 218  				LogPrint (eLogError, "NetDb: Runtime exception: ", ex.what ());
 219  			}
 220  		}
 221  	}
 222  
 223  	std::shared_ptr<const RouterInfo> NetDb::AddRouterInfo (const uint8_t * buf, int len)
 224  	{
 225  		bool updated;
 226  		return AddRouterInfo (buf, len, updated);
 227  	}
 228  
 229  	std::shared_ptr<const RouterInfo> NetDb::AddRouterInfo (const uint8_t * buf, int len, bool& updated)
 230  	{
 231  		IdentityEx identity;
 232  		if (identity.FromBuffer (buf, len))
 233  			return AddRouterInfo (identity.GetIdentHash (), buf, len, updated);
 234  		updated = false;
 235  		return nullptr;
 236  	}
 237  
 238  	bool NetDb::AddRouterInfo (const IdentHash& ident, const uint8_t * buf, int len)
 239  	{
 240  		bool updated;
 241  		if (!AddRouterInfo (ident, buf, len, updated))
 242  			updated = false;
 243  		return updated;
 244  	}
 245  
 246  	std::shared_ptr<const RouterInfo> NetDb::AddRouterInfo (const IdentHash& ident, const uint8_t * buf, int len, bool& updated)
 247  	{
 248  		updated = true;
 249  		auto r = FindRouter (ident);
 250  		if (r)
 251  		{
 252  			if (r->IsNewer (buf, len))
 253  			{
 254  				bool wasFloodfill = r->IsFloodfill ();
 255  				{
 256  					std::lock_guard<std::mutex> l(m_RouterInfosMutex);
 257  					if (!r->Update (buf, len))
 258  					{
 259  						updated = false;
 260  						m_Requests->RequestComplete (ident, r);
 261  						return r;
 262  					}
 263  					if (r->IsUnreachable () ||
 264  					    i2p::util::GetMillisecondsSinceEpoch () + NETDB_EXPIRATION_TIMEOUT_THRESHOLD*1000LL < r->GetTimestamp ())
 265  					{
 266  						// delete router as invalid or from future after update
 267  						m_RouterInfos.erase (ident);
 268  						if (wasFloodfill)
 269  						{
 270  							std::lock_guard<std::mutex> l(m_FloodfillsMutex);
 271  							m_Floodfills.Remove (r->GetIdentHash ());
 272  						}
 273  						m_Requests->RequestComplete (ident, nullptr);
 274  						return nullptr;
 275  					}
 276  				}
 277  				if (CheckLogLevel (eLogInfo))
 278  					LogPrint (eLogInfo, "NetDb: RouterInfo updated: ", ident.ToBase64());
 279  				if (wasFloodfill != r->IsFloodfill ()) // if floodfill status updated
 280  				{
 281  					if (CheckLogLevel (eLogDebug))
 282  						LogPrint (eLogDebug, "NetDb: RouterInfo floodfill status updated: ", ident.ToBase64());
 283  					std::lock_guard<std::mutex> l(m_FloodfillsMutex);
 284  					if (wasFloodfill)
 285  						m_Floodfills.Remove (r->GetIdentHash ());
 286  					else if (r->IsEligibleFloodfill ())
 287  					{
 288  						if (m_Floodfills.GetSize () < NETDB_NUM_FLOODFILLS_THRESHOLD || r->GetProfile ()->IsReal ())
 289  							m_Floodfills.Insert (r);
 290  						else
 291  							r->ResetFloodfill ();
 292  					}
 293  				}
 294  			}
 295  			else
 296  			{
 297  				r->CancelBufferToDelete (); // since an update received
 298  				if (CheckLogLevel (eLogDebug))
 299  					LogPrint (eLogDebug, "NetDb: RouterInfo is older: ", ident.ToBase64());
 300  				updated = false;
 301  			}
 302  		}
 303  		else
 304  		{
 305  			r = std::make_shared<RouterInfo> (buf, len);
 306  			bool isValid = !r->IsUnreachable () && r->HasValidAddresses () && (!r->IsFloodfill () || !r->GetProfile ()->IsUnreachable ());
 307  			if (isValid)
 308  			{
 309  				auto mts = i2p::util::GetMillisecondsSinceEpoch ();
 310  			    isValid = mts + NETDB_EXPIRATION_TIMEOUT_THRESHOLD*1000LL > r->GetTimestamp () && // from future
 311  					(mts < r->GetTimestamp () + NETDB_MAX_EXPIRATION_TIMEOUT*1000LL || // too old
 312  					 context.GetUptime () < NETDB_CHECK_FOR_EXPIRATION_UPTIME/10); // enough uptime
 313  			}
 314  			if (isValid)
 315  			{
 316  				bool inserted = false;
 317  				{
 318  					std::lock_guard<std::mutex> l(m_RouterInfosMutex);
 319  					inserted = m_RouterInfos.insert ({r->GetIdentHash (), r}).second;
 320  				}
 321  				if (inserted)
 322  				{
 323  					if (CheckLogLevel (eLogInfo))
 324  						LogPrint (eLogInfo, "NetDb: RouterInfo added: ", ident.ToBase64());
 325  					if (r->IsFloodfill () && r->IsEligibleFloodfill ())
 326  					{
 327  						if (m_Floodfills.GetSize () < NETDB_NUM_FLOODFILLS_THRESHOLD ||
 328  						 r->GetProfile ()->IsReal ()) // don't insert floodfill until it's known real if we have enough
 329  						{
 330  							std::lock_guard<std::mutex> l(m_FloodfillsMutex);
 331  							m_Floodfills.Insert (r);
 332  						}
 333  						else
 334  							r->ResetFloodfill ();
 335  					}
 336  				}
 337  				else
 338  				{
 339  					LogPrint (eLogWarning, "NetDb: Duplicated RouterInfo ", ident.ToBase64());
 340  					updated = false;
 341  				}
 342  			}
 343  			else
 344  				updated = false;
 345  		}
 346  		// take care about requested destination
 347  		m_Requests->RequestComplete (ident, r);
 348  		return r;
 349  	}
 350  
 351  	bool NetDb::AddLeaseSet (const IdentHash& ident, const uint8_t * buf, int len)
 352  	{
 353  		std::lock_guard<std::mutex> lock(m_LeaseSetsMutex);
 354  		bool updated = false;
 355  		auto it = m_LeaseSets.find(ident);
 356  		if (it != m_LeaseSets.end () && it->second->GetStoreType () == i2p::data::NETDB_STORE_TYPE_LEASESET)
 357  		{
 358  			// we update only is existing LeaseSet is not LeaseSet2
 359  			uint64_t expires;
 360  			if(LeaseSetBufferValidate(buf, len, expires))
 361  			{
 362  				if(it->second->GetExpirationTime() < expires)
 363  				{
 364  					it->second->Update (buf, len, nullptr, false); // signature is verified already
 365  					if (CheckLogLevel (eLogInfo))
 366  						LogPrint (eLogInfo, "NetDb: LeaseSet updated: ", ident.ToBase32());
 367  					updated = true;
 368  				}
 369  				else if (CheckLogLevel (eLogDebug))
 370  					LogPrint(eLogDebug, "NetDb: LeaseSet is older: ", ident.ToBase32());
 371  			}
 372  			else
 373  				LogPrint(eLogError, "NetDb: LeaseSet is invalid: ", ident.ToBase32());
 374  		}
 375  		else
 376  		{
 377  			auto leaseSet = std::make_shared<LeaseSet> (buf, len, false); // we don't need leases in netdb
 378  			if (leaseSet->IsValid ())
 379  			{
 380  				if (CheckLogLevel (eLogInfo))
 381  					LogPrint (eLogInfo, "NetDb: LeaseSet added: ", ident.ToBase32());
 382  				m_LeaseSets[ident] = leaseSet;
 383  				updated = true;
 384  			}
 385  			else
 386  				LogPrint (eLogError, "NetDb: New LeaseSet validation failed: ", ident.ToBase32());
 387  		}
 388  		return updated;
 389  	}
 390  
 391  	bool NetDb::AddLeaseSet2 (const IdentHash& ident, const uint8_t * buf, int len, uint8_t storeType)
 392  	{
 393  		auto leaseSet = std::make_shared<LeaseSet2> (storeType, buf, len, false); // we don't need leases in netdb
 394  		if (leaseSet->IsValid ())
 395  		{
 396  			std::lock_guard<std::mutex> lock(m_LeaseSetsMutex);
 397  			auto it = m_LeaseSets.find(ident);
 398  			if (it == m_LeaseSets.end () || it->second->GetStoreType () != storeType ||
 399  				leaseSet->GetPublishedTimestamp () > it->second->GetPublishedTimestamp ())
 400  			{
 401  				if (leaseSet->IsPublic () && !leaseSet->IsExpired ())
 402  				{
 403  					// TODO: implement actual update
 404  					if (CheckLogLevel (eLogInfo))
 405  						LogPrint (eLogInfo, "NetDb: LeaseSet2 updated: ", ident.ToBase32());
 406  					m_LeaseSets[ident] = leaseSet;
 407  					return true;
 408  				}
 409  				else
 410  				{
 411  					LogPrint (eLogWarning, "NetDb: Unpublished or expired or future LeaseSet2 received: ", ident.ToBase32());
 412  					m_LeaseSets.erase (ident);
 413  				}
 414  			}
 415  		}
 416  		else
 417  			LogPrint (eLogError, "NetDb: New LeaseSet2 validation failed: ", ident.ToBase32());
 418  		return false;
 419  	}
 420  
 421  	std::shared_ptr<RouterInfo> NetDb::FindRouter (const IdentHash& ident) const
 422  	{
 423  		std::lock_guard<std::mutex> l(m_RouterInfosMutex);
 424  		auto it = m_RouterInfos.find (ident);
 425  		if (it != m_RouterInfos.end ())
 426  			return it->second;
 427  		else
 428  			return nullptr;
 429  	}
 430  
 431  	std::shared_ptr<LeaseSet> NetDb::FindLeaseSet (const IdentHash& destination) const
 432  	{
 433  		std::lock_guard<std::mutex> lock(m_LeaseSetsMutex);
 434  		auto it = m_LeaseSets.find (destination);
 435  		if (it != m_LeaseSets.end ())
 436  			return it->second;
 437  		else
 438  			return nullptr;
 439  	}
 440  
 441  	std::shared_ptr<RouterProfile> NetDb::FindRouterProfile (const IdentHash& ident) const
 442  	{
 443  		if (!m_PersistProfiles)
 444  			return nullptr;
 445  
 446  		auto router = FindRouter (ident);
 447  		return router ? router->GetProfile () : nullptr;
 448  	}
 449  
 450  	void NetDb::SetUnreachable (const IdentHash& ident, bool unreachable)
 451  	{
 452  		auto r = FindRouter (ident);
 453  		if (r)
 454  		{
 455  			r->SetUnreachable (unreachable);
 456  			auto profile = r->GetProfile ();
 457  			if (profile)
 458  			{
 459  				profile->Unreachable (unreachable);
 460  				if (!unreachable && r->IsDeclaredFloodfill () && !r->IsFloodfill () &&
 461  				    r->IsEligibleFloodfill () && profile->IsReal ())
 462  				{
 463  					// enable previously disabled floodfill
 464  					r->SetFloodfill ();
 465  					std::lock_guard<std::mutex> l(m_FloodfillsMutex);
 466  					m_Floodfills.Insert (r);
 467  				}
 468  			}
 469  		}
 470  	}
 471  
 472  	void NetDb::ExcludeReachableTransports (const IdentHash& ident, RouterInfo::CompatibleTransports transports)
 473  	{
 474  		auto r = FindRouter (ident);
 475  		if (r)
 476  		{
 477  			std::lock_guard<std::mutex> l(m_RouterInfosMutex);
 478  			r->ExcludeReachableTransports (transports);
 479  		}
 480  	}
 481  
 482  	void NetDb::Reseed ()
 483  	{
 484  		if (!m_Reseeder)
 485  		{
 486  			m_Reseeder = new Reseeder ();
 487  			m_Reseeder->LoadCertificates (); // we need certificates for SU3 verification
 488  		}
 489  
 490  		m_Reseeder->Bootstrap ();
 491  	}
 492  
 493  	void NetDb::ReseedFromFloodfill(const RouterInfo & ri, int numRouters, int numFloodfills)
 494  	{
 495  		LogPrint(eLogInfo, "NetDB: Reseeding from floodfill ", ri.GetIdentHashBase64());
 496  		std::list<std::shared_ptr<i2p::I2NPMessage> > requests;
 497  
 498  		i2p::data::IdentHash ourIdent = i2p::context.GetIdentHash();
 499  		i2p::data::IdentHash ih = ri.GetIdentHash();
 500  		i2p::data::IdentHash randomIdent;
 501  
 502  		// make floodfill lookups
 503  		while(numFloodfills > 0) {
 504  			randomIdent.Randomize();
 505  			auto msg = i2p::CreateRouterInfoDatabaseLookupMsg(randomIdent, ourIdent, 0, false);
 506  			requests.push_back(msg);
 507  			numFloodfills --;
 508  		}
 509  
 510  		// make regular router lookups
 511  		while(numRouters > 0) {
 512  			randomIdent.Randomize();
 513  			auto msg = i2p::CreateRouterInfoDatabaseLookupMsg(randomIdent, ourIdent, 0, true);
 514  			requests.push_back(msg);
 515  			numRouters --;
 516  		}
 517  
 518  		// send them off
 519  		i2p::transport::transports.SendMessages(ih, std::move (requests));
 520  	}
 521  
 522  	bool NetDb::LoadRouterInfo (const std::string& path, uint64_t ts)
 523  	{
 524  		auto r = std::make_shared<RouterInfo>(path);
 525  		if (r->GetRouterIdentity () && !r->IsUnreachable () && r->HasValidAddresses () &&
 526  			ts < r->GetTimestamp () + 24*60*60*NETDB_MAX_OFFLINE_EXPIRATION_TIMEOUT*1000LL && // too old
 527  			(r->GetVersion () >= NETDB_MIN_ALLOWED_VERSION || r->IsHighBandwidth ())) // old version
 528  		{
 529  			r->DeleteBuffer ();
 530  			if (m_RouterInfos.emplace (r->GetIdentHash (), r).second)
 531  			{
 532  				if (r->IsFloodfill () && r->IsEligibleFloodfill ())
 533  					m_Floodfills.Insert (r);
 534  			}
 535  		}
 536  		else
 537  		{
 538  			LogPrint(eLogWarning, "NetDb: RI from ", path, " is invalid or too old. Delete");
 539  			i2p::fs::Remove(path);
 540  		}
 541  		return true;
 542  	}
 543  
 544  	void NetDb::VisitLeaseSets(LeaseSetVisitor v)
 545  	{
 546  		std::lock_guard<std::mutex> lock(m_LeaseSetsMutex);
 547  		for ( auto & entry : m_LeaseSets)
 548  			v(entry.first, entry.second);
 549  	}
 550  
 551  	void NetDb::VisitStoredRouterInfos(RouterInfoVisitor v)
 552  	{
 553  		m_Storage.Iterate([v] (const std::string & filename)
 554  		{
 555  			auto ri = std::make_shared<i2p::data::RouterInfo>(filename);
 556  				v(ri);
 557  		});
 558  	}
 559  
 560  	void NetDb::VisitRouterInfos(RouterInfoVisitor v)
 561  	{
 562  		std::lock_guard<std::mutex> lock(m_RouterInfosMutex);
 563  		for ( const auto & item : m_RouterInfos )
 564  			v(item.second);
 565  	}
 566  
 567  	size_t NetDb::VisitRandomRouterInfos(RouterInfoFilter filter, RouterInfoVisitor v, size_t n)
 568  	{
 569  		std::vector<std::shared_ptr<const RouterInfo> > found;
 570  		const size_t max_iters_per_cyle = 3;
 571  		size_t iters = max_iters_per_cyle;
 572  		while(n > 0)
 573  		{
 574  			std::lock_guard<std::mutex> lock(m_RouterInfosMutex);
 575  			uint32_t idx = m_Rng () % m_RouterInfos.size ();
 576  			uint32_t i = 0;
 577  			for (const auto & it : m_RouterInfos) {
 578  				if(i >= idx) // are we at the random start point?
 579  				{
 580  					// yes, check if we want this one
 581  					if(filter(it.second))
 582  					{
 583  						// we have a match
 584  						--n;
 585  						found.push_back(it.second);
 586  						// reset max iterations per cycle
 587  						iters = max_iters_per_cyle;
 588  						break;
 589  					}
 590  				}
 591  				else // not there yet
 592  					++i;
 593  			}
 594  			// we have enough
 595  			if(n == 0) break;
 596  			--iters;
 597  			// have we tried enough this cycle ?
 598  			if(!iters) {
 599  				// yes let's try the next cycle
 600  				--n;
 601  				iters = max_iters_per_cyle;
 602  			}
 603  		}
 604  		// visit the ones we found
 605  		size_t visited = 0;
 606  		for(const auto & ri : found ) {
 607  			v(ri);
 608  			++visited;
 609  		}
 610  		return visited;
 611  	}
 612  
 613  	void NetDb::Load ()
 614  	{
 615  		// make sure we cleanup netDb from previous attempts
 616  		m_RouterInfos.clear ();
 617  		m_Floodfills.Clear ();
 618  
 619  		uint64_t ts = i2p::util::GetMillisecondsSinceEpoch();
 620  		std::vector<std::string> files;
 621  		m_Storage.Traverse(files);
 622  		for (const auto& path : files)
 623  			LoadRouterInfo (path, ts);
 624  
 625  		LogPrint (eLogInfo, "NetDb: ", m_RouterInfos.size(), " routers loaded (", m_Floodfills.GetSize (), " floodfils)");
 626  	}
 627  
 628  	void NetDb::SaveUpdated ()
 629  	{
 630  		if (m_PersistingRouters.valid ())
 631  		{
 632  			if (m_PersistingRouters.wait_for(std::chrono::seconds(0)) == std::future_status::ready)
 633  				m_PersistingRouters.get ();
 634  			else
 635  			{
 636  				LogPrint (eLogWarning, "NetDb: Can't save updated routers. Routers are being saved to disk");
 637  				return;
 638  			}
 639  		}
 640  
 641  		int updatedCount = 0, deletedCount = 0, deletedFloodfillsCount = 0;
 642  		auto total = m_RouterInfos.size ();
 643  		auto totalFloodfills = m_Floodfills.GetSize ();
 644  		uint64_t expirationTimeout = NETDB_MAX_EXPIRATION_TIMEOUT*1000LL;
 645  		uint64_t ts = i2p::util::GetMillisecondsSinceEpoch();
 646  		auto uptime = i2p::context.GetUptime ();
 647  		double minTunnelCreationSuccessRate;
 648  		i2p::config::GetOption("limits.zombies", minTunnelCreationSuccessRate);
 649  		bool isLowRate = i2p::tunnel::tunnels.GetPreciseTunnelCreationSuccessRate () < minTunnelCreationSuccessRate;
 650  		// routers don't expire if less than 90 or uptime is less than 1 hour
 651  		bool checkForExpiration = total > NETDB_MIN_ROUTERS && uptime > NETDB_CHECK_FOR_EXPIRATION_UPTIME; // 10 minutes
 652  		if (checkForExpiration && uptime > i2p::transport::SSU2_TO_INTRODUCER_SESSION_DURATION) // 1 hour
 653  			expirationTimeout = i2p::context.IsFloodfill () ? NETDB_FLOODFILL_EXPIRATION_TIMEOUT*1000LL :
 654  				NETDB_MIN_EXPIRATION_TIMEOUT*1000LL + (NETDB_MAX_EXPIRATION_TIMEOUT - NETDB_MIN_EXPIRATION_TIMEOUT)*1000LL*NETDB_MIN_ROUTERS/total;
 655  		bool isOffline = checkForExpiration && i2p::transport::transports.GetNumPeers () < NETDB_MIN_TRANSPORTS; // enough routers and uptime, but no transports
 656  
 657  		std::list<std::pair<std::string, std::shared_ptr<RouterInfo::Buffer> > > saveToDisk;
 658  		std::list<std::string> removeFromDisk;
 659  
 660  		auto own = i2p::context.GetSharedRouterInfo ();
 661  		for (auto [ident, r]: m_RouterInfos)
 662  		{
 663  			if (!r || r == own) continue; // skip own
 664  			if (r->IsBufferScheduledToDelete ()) // from previous SaveUpdated, we assume m_PersistingRouters complete
 665  			{
 666  				std::lock_guard<std::mutex> l(m_RouterInfosMutex); // possible collision between DeleteBuffer and Update
 667  				r->DeleteBuffer ();
 668  			}
 669  			if (r->IsUpdated ())
 670  			{
 671  				if (r->GetBuffer () && !r->IsUnreachable ())
 672  				{
 673  					// we have something to save
 674  					std::shared_ptr<RouterInfo::Buffer> buffer;
 675  					{
 676  						std::lock_guard<std::mutex> l(m_RouterInfosMutex); // possible collision between DeleteBuffer and Update
 677  						buffer = r->CopyBuffer ();
 678  					}
 679  					if (!i2p::transport::transports.IsConnected (ident))
 680  						r->ScheduleBufferToDelete ();
 681  					if (buffer)
 682  						saveToDisk.emplace_back(ident.ToBase64 (), buffer);
 683  				}
 684  				r->SetUpdated (false);
 685  				updatedCount++;
 686  				continue;
 687  			}
 688  			else if (r->GetBuffer () && ts > r->GetTimestamp () + NETDB_MIN_EXPIRATION_TIMEOUT*1000LL)
 689  				// since update was long time ago we assume that router is not connected anymore
 690  				r->ScheduleBufferToDelete ();
 691  
 692  			if (r->HasProfile () && r->GetProfile ()->IsUnreachable ())
 693  				r->SetUnreachable (true);
 694  			// make router reachable back if too few routers or floodfills
 695  			if (r->IsUnreachable () && (total - deletedCount < NETDB_MIN_ROUTERS || isLowRate || isOffline ||
 696  				(r->IsFloodfill () && totalFloodfills - deletedFloodfillsCount < NETDB_MIN_FLOODFILLS)))
 697  				r->SetUnreachable (false);
 698  			if (!r->IsUnreachable ())
 699  			{
 700  				// find & mark expired routers
 701  				if (!r->GetCompatibleTransports (true)) // non reachable by any transport
 702  					r->SetUnreachable (true);
 703  				else if (r->GetVersion () < NETDB_MIN_ALLOWED_VERSION && !r->IsHighBandwidth ()) // too old
 704  					r->SetUnreachable (true);
 705  				else if (ts + NETDB_EXPIRATION_TIMEOUT_THRESHOLD*1000LL < r->GetTimestamp ())
 706  				{
 707  					LogPrint (eLogWarning, "NetDb: RouterInfo is from future for ", (r->GetTimestamp () - ts)/1000LL, " seconds");
 708  					r->SetUnreachable (true);
 709  				}
 710  				else if (checkForExpiration)
 711  				{
 712  					if (ts > r->GetTimestamp () + expirationTimeout)
 713  						r->SetUnreachable (true);
 714  					else if ((ts > r->GetTimestamp () + expirationTimeout/2) && // more than half of expiration
 715  						total > NETDB_NUM_ROUTERS_THRESHOLD && !r->IsHighBandwidth() &&  // low bandwidth
 716  						!r->IsFloodfill() && (!i2p::context.IsFloodfill () || // non floodfill
 717  					    (CreateRoutingKey (ident) ^ i2p::context.GetIdentHash ()).metric[0] >= 0x02)) // different first 7 bits
 718  							r->SetUnreachable (true);
 719  				}
 720  			}
 721  			// make router reachable back if connected now or trusted router
 722  			if (r->IsUnreachable () && (i2p::transport::transports.IsConnected (ident) ||
 723  				i2p::transport::transports.IsTrustedRouter (ident)))
 724  				r->SetUnreachable (false);
 725  
 726  			if (r->IsUnreachable ())
 727  			{
 728  				if (r->IsFloodfill ()) deletedFloodfillsCount++;
 729  				// delete RI file
 730  				removeFromDisk.emplace_back (ident.ToBase64());
 731  				deletedCount++;
 732  				if (total - deletedCount < NETDB_MIN_ROUTERS) checkForExpiration = false;
 733  			}
 734  		} // m_RouterInfos iteration
 735  
 736  		if (!saveToDisk.empty () || !removeFromDisk.empty ())
 737  		{
 738  			m_PersistingRouters = std::async (std::launch::async, &NetDb::PersistRouters,
 739  				this, std::move (saveToDisk), std::move (removeFromDisk));
 740  		}
 741  
 742  		m_RouterInfoBuffersPool.CleanUpMt ();
 743  		m_RouterInfoAddressesPool.CleanUpMt ();
 744  		m_RouterInfoAddressVectorsPool.CleanUpMt ();
 745  		m_IdentitiesPool.CleanUpMt ();
 746  
 747  		if (updatedCount > 0)
 748  			LogPrint (eLogInfo, "NetDb: Saved ", updatedCount, " new/updated routers");
 749  		if (deletedCount > 0)
 750  		{
 751  			LogPrint (eLogInfo, "NetDb: Deleting ", deletedCount, " unreachable routers");
 752  			// clean up RouterInfos table
 753  			{
 754  				std::lock_guard<std::mutex> l(m_RouterInfosMutex);
 755  				for (auto it = m_RouterInfos.begin (); it != m_RouterInfos.end ();)
 756  				{
 757  					if (!it->second || it->second->IsUnreachable ())
 758  						it = m_RouterInfos.erase (it);
 759  					else
 760  					{
 761  						it->second->DropProfile ();
 762  						it++;
 763  					}
 764  				}
 765  			}
 766  			// clean up expired floodfills or not floodfills anymore
 767  			{
 768  				std::lock_guard<std::mutex> l(m_FloodfillsMutex);
 769  				m_Floodfills.Cleanup ([](const std::shared_ptr<RouterInfo>& r)->bool
 770  					{
 771  						return r && r->IsFloodfill () && !r->IsUnreachable ();
 772  					});
 773  			}
 774  		}
 775  	}
 776  
 777  	void NetDb::PersistRouters (std::list<std::pair<std::string, std::shared_ptr<RouterInfo::Buffer> > >&& update,
 778  		std::list<std::string>&& remove)
 779  	{
 780  		for (auto it: update)
 781  			RouterInfo::SaveToFile (m_Storage.Path(it.first), it.second);
 782  		for (auto it: remove)
 783  			m_Storage.Remove (it);
 784  	}
 785  
 786  	void NetDb::RequestDestination (const IdentHash& destination, RequestedDestination::RequestComplete requestComplete, bool direct)
 787  	{
 788  		if (direct && (i2p::transport::transports.RoutesRestricted () || i2p::context.IsLimitedConnectivity ()))
 789  		    direct = false; // always use tunnels for restricted routes or limited connectivity
 790  		if (m_Requests)
 791  			m_Requests->PostRequestDestination (destination, requestComplete, direct);
 792  		else
 793  			LogPrint (eLogError, "NetDb: Requests is null");
 794  	}
 795  
 796  	void NetDb::HandleNTCP2RouterInfoMsg (std::shared_ptr<const I2NPMessage> m)
 797  	{
 798  		uint8_t flood = m->GetPayload ()[0] & NTCP2_ROUTER_INFO_FLAG_REQUEST_FLOOD;
 799  		bool updated;
 800  		auto ri = AddRouterInfo (m->GetPayload () + 1, m->GetPayloadLength () - 1, updated); // without flags
 801  		if (flood && updated && context.IsFloodfill () && ri)
 802  		{
 803  			auto floodMsg = CreateDatabaseStoreMsg (ri, 0); // replyToken = 0
 804  			Flood (ri->GetIdentHash (), floodMsg);
 805  		}
 806  	}
 807  
 808  	void NetDb::HandleDatabaseStoreMsg (std::shared_ptr<const I2NPMessage> m)
 809  	{
 810  		const uint8_t * buf = m->GetPayload ();
 811  		size_t len = m->GetSize ();
 812  		if (len < DATABASE_STORE_HEADER_SIZE)
 813  		{
 814  			LogPrint (eLogError, "NetDb: Database store msg is too short ", len, ". Dropped");
 815  			return;
 816  		}
 817  		IdentHash ident (buf + DATABASE_STORE_KEY_OFFSET);
 818  		if (ident.IsZero ())
 819  		{
 820  			LogPrint (eLogDebug, "NetDb: Database store with zero ident, dropped");
 821  			return;
 822  		}
 823  		uint32_t replyToken = bufbe32toh (buf + DATABASE_STORE_REPLY_TOKEN_OFFSET);
 824  		size_t offset = DATABASE_STORE_HEADER_SIZE;
 825  		if (replyToken)
 826  		{
 827  			if (len < offset + 36) // 32 + 4
 828  			{
 829  				LogPrint (eLogError, "NetDb: Database store msg with reply token is too short ", len, ". Dropped");
 830  				return;
 831  			}
 832  			uint32_t tunnelID = bufbe32toh (buf + offset);
 833  			offset += 4;
 834  			if (replyToken != 0xFFFFFFFFU) // if not caught on OBEP or IBGW
 835  			{
 836  				IdentHash replyIdent(buf + offset);
 837  				auto deliveryStatus = CreateDeliveryStatusMsg (replyToken);
 838  				if (!tunnelID) // send response directly
 839  					transports.SendMessage (replyIdent, deliveryStatus);
 840  				else
 841  				{
 842  					bool direct = true;
 843  					if (!i2p::transport::transports.IsConnected (replyIdent))
 844  					{
 845  						auto r = FindRouter (replyIdent);
 846  						if (r && !r->IsReachableFrom (i2p::context.GetRouterInfo ()))
 847  							direct = false;
 848  					}
 849  					if (direct) // send response directly to IBGW
 850  						transports.SendMessage (replyIdent, i2p::CreateTunnelGatewayMsg (tunnelID, deliveryStatus));
 851  					else
 852  					{
 853  						// send response through exploratory tunnel
 854  						auto pool = i2p::tunnel::tunnels.GetExploratoryPool ();
 855  						auto outbound = pool ? pool->GetNextOutboundTunnel () : nullptr;
 856  						if (outbound)
 857  							outbound->SendTunnelDataMsgTo (replyIdent, tunnelID, deliveryStatus);
 858  						else
 859  							LogPrint (eLogWarning, "NetDb: No outbound tunnels for DatabaseStore reply found");
 860  					}
 861  				}
 862  			}
 863  			offset += 32;
 864  		}
 865  		// we must send reply back before this check
 866  		if (ident == i2p::context.GetIdentHash ())
 867  		{
 868  			LogPrint (eLogDebug, "NetDb: Database store with own RouterInfo received, dropped");
 869  			return;
 870  		}
 871  		size_t payloadOffset = offset;
 872  
 873  		bool updated = false;
 874  		uint8_t storeType = buf[DATABASE_STORE_TYPE_OFFSET];
 875  		if (storeType) // LeaseSet or LeaseSet2
 876  		{
 877  			if (len > MAX_LS_BUFFER_SIZE + offset)
 878  			{
 879  				LogPrint (eLogError, "NetDb: Database store message is too long ", len);
 880  				return;
 881  			}
 882  			if (!context.IsFloodfill ())
 883  			{
 884  				LogPrint (eLogInfo, "NetDb: Not Floodfill, LeaseSet store request ignored for ", ident.ToBase32());
 885  				return;
 886  			}
 887  			else if (!m->from) // unsolicited LS must be received directly
 888  			{
 889  				if (storeType == NETDB_STORE_TYPE_LEASESET) // 1
 890  				{
 891  					if (CheckLogLevel (eLogDebug))
 892  						LogPrint (eLogDebug, "NetDb: Store request: LeaseSet for ", ident.ToBase32());
 893  					updated = AddLeaseSet (ident, buf + offset, len - offset);
 894  				}
 895  				else // all others are considered as LeaseSet2
 896  				{
 897  					if (CheckLogLevel (eLogDebug))
 898  						LogPrint (eLogDebug, "NetDb: Store request: LeaseSet2 of type ", int(storeType), " for ", ident.ToBase32());
 899  					updated = AddLeaseSet2 (ident, buf + offset, len - offset, storeType);
 900  				}
 901  			}
 902  		}
 903  		else // RouterInfo
 904  		{
 905  			if (CheckLogLevel (eLogDebug))
 906  				LogPrint (eLogDebug, "NetDb: Store request: RouterInfo ", ident.ToBase64());
 907  			size_t size = bufbe16toh (buf + offset);
 908  			offset += 2;
 909  			if (size > MAX_RI_BUFFER_SIZE || size > len - offset)
 910  			{
 911  				LogPrint (eLogError, "NetDb: Invalid RouterInfo length ", (int)size);
 912  				return;
 913  			}
 914  			uint8_t uncompressed[MAX_RI_BUFFER_SIZE];
 915  			size_t uncompressedSize = m_Inflator.Inflate (buf + offset, size, uncompressed, MAX_RI_BUFFER_SIZE);
 916  			if (uncompressedSize && uncompressedSize < MAX_RI_BUFFER_SIZE)
 917  				updated = AddRouterInfo (ident, uncompressed, uncompressedSize);
 918  			else
 919  			{
 920  				LogPrint (eLogInfo, "NetDb: Decompression failed ", uncompressedSize);
 921  				return;
 922  			}
 923  		}
 924  
 925  		if (replyToken && context.IsFloodfill () && updated)
 926  		{
 927  			// flood updated
 928  			auto floodMsg = NewI2NPShortMessage ();
 929  			uint8_t * payload = floodMsg->GetPayload ();
 930  			memcpy (payload, buf, 33); // key + type
 931  			htobe32buf (payload + DATABASE_STORE_REPLY_TOKEN_OFFSET, 0); // zero reply token
 932  			size_t msgLen = len - payloadOffset;
 933  			floodMsg->len += DATABASE_STORE_HEADER_SIZE + msgLen;
 934  			if (floodMsg->len < floodMsg->maxLen)
 935  			{
 936  				memcpy (payload + DATABASE_STORE_HEADER_SIZE, buf + payloadOffset, msgLen);
 937  				floodMsg->FillI2NPMessageHeader (eI2NPDatabaseStore);
 938  				int minutesBeforeMidnight = 24*60 - i2p::util::GetMinutesSinceEpoch () % (24*60);
 939  				bool andNextDay = storeType ? minutesBeforeMidnight < NETDB_NEXT_DAY_LEASESET_THRESHOLD:
 940  					minutesBeforeMidnight < NETDB_NEXT_DAY_ROUTER_INFO_THRESHOLD;
 941  				Flood (ident, floodMsg, andNextDay);
 942  			}
 943  			else
 944  				LogPrint (eLogError, "NetDb: Database store message is too long ", floodMsg->len);
 945  		}
 946  	}
 947  
 948  	void NetDb::HandleDatabaseLookupMsg (std::shared_ptr<const I2NPMessage> msg)
 949  	{
 950  		const uint8_t * buf = msg->GetPayload ();
 951  		IdentHash ident (buf);
 952  		if (ident.IsZero ())
 953  		{
 954  			LogPrint (eLogError, "NetDb: DatabaseLookup for zero ident. Ignored");
 955  			return;
 956  		}
 957  		std::string key;
 958  		if (CheckLogLevel (eLogInfo))
 959  			key = i2p::data::ByteStreamToBase64 (buf, 32);
 960  
 961  		IdentHash replyIdent(buf + 32);
 962  		uint8_t flag = buf[64];
 963  
 964  		LogPrint (eLogDebug, "NetDb: DatabaseLookup for ", key, " received flags=", (int)flag);
 965  		uint8_t lookupType = flag & DATABASE_LOOKUP_TYPE_FLAGS_MASK;
 966  		const uint8_t * excluded = buf + 65;
 967  		uint32_t replyTunnelID = 0;
 968  		if (flag & DATABASE_LOOKUP_DELIVERY_FLAG) //reply to tunnel
 969  		{
 970  			replyTunnelID = bufbe32toh (excluded);
 971  			excluded += 4;
 972  		}
 973  		uint16_t numExcluded = bufbe16toh (excluded);
 974  		excluded += 2;
 975  		if (numExcluded > 512 || (excluded - buf) + numExcluded*32 > (int)msg->GetPayloadLength ())
 976  		{
 977  			LogPrint (eLogWarning, "NetDb: Number of excluded peers", numExcluded, " is too much");
 978  			return;
 979  		}
 980  
 981  		std::shared_ptr<I2NPMessage> replyMsg;
 982  		if (lookupType == DATABASE_LOOKUP_TYPE_EXPLORATORY_LOOKUP)
 983  		{
 984  			if (!context.IsFloodfill ())
 985  			{
 986  				LogPrint (eLogWarning, "NetDb: Exploratory lookup to non-floodfill dropped");
 987  				return;
 988  			}
 989  			LogPrint (eLogInfo, "NetDb: Exploratory close to ", key, " ", numExcluded, " excluded");
 990  			std::unordered_set<IdentHash> excludedRouters;
 991  			const uint8_t * excluded_ident = excluded;
 992  			for (int i = 0; i < numExcluded; i++)
 993  			{
 994  				excludedRouters.insert (excluded_ident);
 995  				excluded_ident += 32;
 996  			}
 997  			replyMsg = CreateDatabaseSearchReply (ident, GetExploratoryNonFloodfill (ident,
 998  				NETDB_MAX_NUM_SEARCH_REPLY_PEER_HASHES, excludedRouters));
 999  		}
1000  		else
1001  		{
1002  			if (lookupType == DATABASE_LOOKUP_TYPE_ROUTERINFO_LOOKUP ||
1003  				lookupType == DATABASE_LOOKUP_TYPE_NORMAL_LOOKUP)
1004  			{
1005  				// try to find router
1006  				auto router = FindRouter (ident);
1007  				if (router && !router->IsUnreachable ())
1008  				{
1009  					LogPrint (eLogDebug, "NetDb: Requested RouterInfo ", key, " found");
1010  					if (PopulateRouterInfoBuffer (router))
1011  						replyMsg = CreateDatabaseStoreMsg (router);
1012  				}
1013  			}
1014  
1015  			if (!replyMsg && (lookupType == DATABASE_LOOKUP_TYPE_LEASESET_LOOKUP ||
1016  				lookupType == DATABASE_LOOKUP_TYPE_NORMAL_LOOKUP))
1017  			{
1018  				// try to find leaseset
1019  				if (context.IsFloodfill ())
1020  				{
1021  					auto leaseSet = FindLeaseSet (ident);
1022  					if (!leaseSet)
1023  					{
1024  						// no leaseset found
1025  						LogPrint(eLogDebug, "NetDb: Requested LeaseSet not found for ", ident.ToBase32());
1026  					}
1027  					else if (!leaseSet->IsExpired ()) // we don't send back expired leasesets
1028  					{
1029  						LogPrint (eLogDebug, "NetDb: Requested LeaseSet ", key, " found");
1030  						replyMsg = CreateDatabaseStoreMsg (ident, leaseSet);
1031  					}
1032  				}
1033  				else if (lookupType == DATABASE_LOOKUP_TYPE_LEASESET_LOOKUP)
1034  				{
1035  					LogPrint (eLogWarning, "NetDb: Explicit LeaseSet lookup to non-floodfill dropped");
1036  					return;
1037  				}
1038  			}
1039  
1040  			if (!replyMsg)
1041  			{
1042  				std::unordered_set<IdentHash> excludedRouters;
1043  				const uint8_t * exclude_ident = excluded;
1044  				for (int i = 0; i < numExcluded; i++)
1045  				{
1046  					excludedRouters.insert (exclude_ident);
1047  					exclude_ident += 32;
1048  				}
1049  				auto closestFloodfills = GetClosestFloodfills (ident, 3, excludedRouters, false);
1050  				if (closestFloodfills.empty ())
1051  					LogPrint (eLogWarning, "NetDb: No more floodfills for ", key, " found. ", numExcluded, " peers excluded");
1052  				replyMsg = CreateDatabaseSearchReply (ident, closestFloodfills);
1053  			}
1054  		}
1055  		excluded += numExcluded * 32;
1056  		if (replyMsg)
1057  		{
1058  			if (replyTunnelID)
1059  			{
1060  				// encryption might be used though tunnel only
1061  				if (flag & (DATABASE_LOOKUP_ENCRYPTION_FLAG | DATABASE_LOOKUP_ECIES_FLAG)) // encrypted reply requested
1062  				{
1063  					const uint8_t * sessionKey = excluded;
1064  					const uint8_t numTags = excluded[32];
1065  					if (numTags)
1066  					{
1067  						if (flag & DATABASE_LOOKUP_ECIES_FLAG)
1068  						{
1069  							uint64_t tag;
1070  							memcpy (&tag, excluded + 33, 8);
1071  							replyMsg = i2p::garlic::WrapECIESX25519Message (replyMsg, sessionKey, tag);
1072  						}
1073  						else
1074  						{
1075  							const i2p::garlic::SessionTag sessionTag(excluded + 33); // take first tag
1076  							i2p::garlic::ElGamalAESSession garlic (sessionKey, sessionTag);
1077  							replyMsg = garlic.WrapSingleMessage (replyMsg);
1078  						}
1079  						if (!replyMsg)
1080  							LogPrint (eLogError, "NetDb: Failed to wrap message");
1081  					}
1082  					else
1083  						LogPrint(eLogWarning, "NetDb: Encrypted reply requested but no tags provided");
1084  				}
1085  				bool direct = true;
1086  				if (!i2p::transport::transports.IsConnected (replyIdent))
1087  				{
1088  					auto r = FindRouter (replyIdent);
1089  					if (r && !r->IsReachableFrom (i2p::context.GetRouterInfo ()))
1090  						direct = false;
1091  				}
1092  				if (direct)
1093  					transports.SendMessage (replyIdent, i2p::CreateTunnelGatewayMsg (replyTunnelID, replyMsg));
1094  				else
1095  				{
1096  					auto exploratoryPool = i2p::tunnel::tunnels.GetExploratoryPool ();
1097  					auto outbound = exploratoryPool ? exploratoryPool->GetNextOutboundTunnel () : nullptr;
1098  					if (outbound)
1099  						outbound->SendTunnelDataMsgTo (replyIdent, replyTunnelID, replyMsg);
1100  					else
1101  						LogPrint (eLogWarning, "NetDb: Can't send lookup reply to ", replyIdent.ToBase64 (), ". Non reachable and no outbound tunnels");
1102  				}
1103  			}
1104  			else
1105  				transports.SendMessage (replyIdent, replyMsg);
1106  		}
1107  	}
1108  
1109  	void NetDb::Flood (const IdentHash& ident, std::shared_ptr<I2NPMessage> floodMsg, bool andNextDay)
1110  	{
1111  		std::unordered_set<IdentHash> excluded;
1112  		excluded.insert (i2p::context.GetIdentHash ()); // don't flood to itself
1113  		excluded.insert (ident); // don't flood back
1114  		for (int i = 0; i < 3; i++)
1115  		{
1116  			auto floodfill = GetClosestFloodfill (ident, excluded, false); // current day
1117  			if (floodfill)
1118  			{
1119  				const auto& h = floodfill->GetIdentHash();
1120  				transports.SendMessage (h, CopyI2NPMessage(floodMsg));
1121  				excluded.insert (h);
1122  			}
1123  			else
1124  				return; // no more floodfills
1125  		}
1126  		if (andNextDay)
1127  		{
1128  			// flood to two more closest flodfills for next day
1129  			std::unordered_set<IdentHash> excluded1;
1130  			excluded1.insert (i2p::context.GetIdentHash ()); // don't flood to itself
1131  			excluded1.insert (ident); // don't flood back
1132  			for (int i = 0; i < 2; i++)
1133  			{
1134  				auto floodfill = GetClosestFloodfill (ident, excluded1, true); // next day
1135  				if (floodfill)
1136  				{
1137  					const auto& h = floodfill->GetIdentHash();
1138  					if (!excluded.count (h)) // we didn't send for current day, otherwise skip
1139  						transports.SendMessage (h, CopyI2NPMessage(floodMsg));
1140  					excluded1.insert (h);
1141  				}
1142  				else
1143  					return;
1144  			}
1145  		}
1146  	}
1147  
1148  	std::shared_ptr<const RouterInfo> NetDb::GetRandomRouter () const
1149  	{
1150  		return GetRandomRouter (
1151  			[](std::shared_ptr<const RouterInfo> router)->bool
1152  			{
1153  				return !router->IsHidden ();
1154  			});
1155  	}
1156  
1157  	std::shared_ptr<const RouterInfo> NetDb::GetRandomRouter (std::shared_ptr<const RouterInfo> compatibleWith,
1158  		bool reverse, bool endpoint, bool clientTunnel) const
1159  	{
1160  		bool checkIsReal = clientTunnel && i2p::tunnel::tunnels.GetPreciseTunnelCreationSuccessRate () < NETDB_TUNNEL_CREATION_RATE_THRESHOLD && // too low rate
1161  			context.GetUptime () > NETDB_CHECK_FOR_EXPIRATION_UPTIME; // after 10 minutes uptime
1162  		return GetRandomRouter (
1163  			[compatibleWith, reverse, endpoint, clientTunnel, checkIsReal](std::shared_ptr<const RouterInfo> router)->bool
1164  			{
1165  				return !router->IsHidden () && router != compatibleWith &&
1166  					(reverse ? (compatibleWith->IsReachableFrom (*router) && router->GetCompatibleTransports (true)):
1167  						router->IsReachableFrom (*compatibleWith)) && !router->IsNAT2NATOnly (*compatibleWith) &&
1168  					router->GetVersion () >= NETDB_MIN_ALLOWED_VERSION &&
1169  					router->IsECIES () && !router->IsHighCongestion (clientTunnel) &&
1170  					(!i2p::transport::transports.IsCheckReserved () || !router->IsSameSubnet (*compatibleWith)) &&
1171  					(!checkIsReal || router->GetProfile ()->IsReal ()) &&
1172  					(!endpoint || (router->IsV4 () && (!reverse || router->IsPublished (true)))); // endpoint must be ipv4 and published if inbound(reverse)
1173  			});
1174  	}
1175  
1176  	std::shared_ptr<const RouterInfo> NetDb::GetRandomSSU2PeerTestRouter (bool v4, const std::unordered_set<IdentHash>& excluded) const
1177  	{
1178  		return GetRandomRouter (
1179  			[v4, &excluded](std::shared_ptr<const RouterInfo> router)->bool
1180  			{
1181  				return !router->IsHidden () && router->IsECIES () &&
1182  					router->IsSSU2PeerTesting (v4) && !excluded.count (router->GetIdentHash ());
1183  			});
1184  	}
1185  
1186  	std::shared_ptr<const RouterInfo> NetDb::GetRandomSSU2Introducer (bool v4, const std::unordered_set<IdentHash>& excluded) const
1187  	{
1188  		return GetRandomRouter (
1189  			[v4, &excluded](std::shared_ptr<const RouterInfo> router)->bool
1190  			{
1191  				return !router->IsHidden () && router->IsSSU2Introducer (v4) &&
1192  					!excluded.count (router->GetIdentHash ());
1193  			});
1194  	}
1195  
1196  	std::shared_ptr<const RouterInfo> NetDb::GetHighBandwidthRandomRouter (std::shared_ptr<const RouterInfo> compatibleWith,
1197  		bool reverse, bool endpoint) const
1198  	{
1199  		bool checkIsReal = i2p::tunnel::tunnels.GetPreciseTunnelCreationSuccessRate () < NETDB_TUNNEL_CREATION_RATE_THRESHOLD && // too low rate
1200  			context.GetUptime () > NETDB_CHECK_FOR_EXPIRATION_UPTIME; // after 10 minutes uptime
1201  		return GetRandomRouter (
1202  			[compatibleWith, reverse, endpoint, checkIsReal](std::shared_ptr<const RouterInfo> router)->bool
1203  			{
1204  				return !router->IsHidden () && router != compatibleWith &&
1205  					(reverse ? (compatibleWith->IsReachableFrom (*router) && router->GetCompatibleTransports (true)) :
1206  						router->IsReachableFrom (*compatibleWith)) && !router->IsNAT2NATOnly (*compatibleWith) &&
1207  					(router->GetCaps () & RouterInfo::eHighBandwidth) &&
1208  					router->GetVersion () >= NETDB_MIN_HIGHBANDWIDTH_VERSION &&
1209  					router->IsECIES () && !router->IsHighCongestion (true) &&
1210  					(!i2p::transport::transports.IsCheckReserved () || !router->IsSameSubnet (*compatibleWith)) &&
1211  					(!checkIsReal || router->GetProfile ()->IsReal ()) &&
1212  					(!endpoint || (router->IsV4 () && (!reverse || router->IsPublished (true)))); // endpoint must be ipv4 and published if inbound(reverse)
1213  
1214  			});
1215  	}
1216  
1217  	template<typename Filter>
1218  	std::shared_ptr<const RouterInfo> NetDb::GetRandomRouter (Filter filter) const
1219  	{
1220  		if (m_RouterInfos.empty())
1221  			return nullptr;
1222  		uint16_t inds[3];
1223  		RAND_bytes ((uint8_t *)inds, sizeof (inds));
1224  		std::lock_guard<std::mutex> l(m_RouterInfosMutex);
1225  		auto count = m_RouterInfos.size ();
1226  		if(count == 0) return nullptr;
1227  		inds[0] %= count;
1228  		auto it = m_RouterInfos.begin ();
1229  		std::advance (it, inds[0]);
1230  		// try random router
1231  		if (it != m_RouterInfos.end () && !it->second->IsUnreachable () && filter (it->second))
1232  			return it->second;
1233  		// try some routers around
1234  		auto it1 = m_RouterInfos.begin ();
1235  		if (inds[0])
1236  		{
1237  			// before
1238  			inds[1] %= inds[0];
1239  			std::advance (it1, (inds[1] + inds[0])/2);
1240  		}
1241  		else
1242  			it1 = it;
1243  		auto it2 = it;
1244  		if (inds[0] < m_RouterInfos.size () - 1)
1245  		{
1246  			// after
1247  			inds[2] %= (m_RouterInfos.size () - 1 - inds[0]); inds[2] /= 2;
1248  			std::advance (it2, inds[2]);
1249  		}
1250  		// it1 - from, it2 - to
1251  		it = it1;
1252  		while (it != it2 && it != m_RouterInfos.end ())
1253  		{
1254  			if (!it->second->IsUnreachable () && filter (it->second))
1255  				return it->second;
1256  			it++;
1257  		}
1258  		// still not found, try from the beginning
1259  		it = m_RouterInfos.begin ();
1260  		while (it != it1 && it != m_RouterInfos.end ())
1261  		{
1262  			if (!it->second->IsUnreachable () && filter (it->second))
1263  				return it->second;
1264  			it++;
1265  		}
1266  		// still not found, try to the beginning
1267  		it = it2;
1268  		while (it != m_RouterInfos.end ())
1269  		{
1270  			if (!it->second->IsUnreachable () && filter (it->second))
1271  				return it->second;
1272  			it++;
1273  		}
1274  		return nullptr; // seems we have too few routers
1275  	}
1276  
1277  	void NetDb::PostI2NPMsg (std::shared_ptr<const I2NPMessage> msg)
1278  	{
1279  		if (msg) m_Queue.Put (msg);
1280  	}
1281  
1282  	void NetDb::PostDatabaseSearchReplyMsg (std::shared_ptr<const I2NPMessage> msg)
1283  	{
1284  		if (msg && m_Requests)
1285  			m_Requests->PostDatabaseSearchReplyMsg (msg);
1286  	}
1287  
1288  	std::shared_ptr<const RouterInfo> NetDb::GetClosestFloodfill (const IdentHash& destination,
1289  		const std::unordered_set<IdentHash>& excluded, bool nextDay) const
1290  	{
1291  		IdentHash destKey = CreateRoutingKey (destination, nextDay);
1292  		std::lock_guard<std::mutex> l(m_FloodfillsMutex);
1293  		return m_Floodfills.FindClosest (destKey, [&excluded](const std::shared_ptr<RouterInfo>& r)->bool
1294  			{
1295  				return r && !r->IsUnreachable () && !r->GetProfile ()->IsUnreachable () &&
1296  					!excluded.count (r->GetIdentHash ());
1297  			});
1298  	}
1299  
1300  	std::vector<IdentHash> NetDb::GetClosestFloodfills (const IdentHash& destination, size_t num,
1301  		std::unordered_set<IdentHash>& excluded, bool closeThanUsOnly) const
1302  	{
1303  		std::vector<IdentHash> res;
1304  		IdentHash destKey = CreateRoutingKey (destination);
1305  		std::vector<std::shared_ptr<RouterInfo> > v;
1306  		{
1307  			std::lock_guard<std::mutex> l(m_FloodfillsMutex);
1308  			v = m_Floodfills.FindClosest (destKey, num, [&excluded](const std::shared_ptr<RouterInfo>& r)->bool
1309  				{
1310  					return r && !r->IsUnreachable () && !r->GetProfile ()->IsUnreachable () &&
1311  						!excluded.count (r->GetIdentHash ());
1312  				});
1313  		}
1314  		if (v.empty ()) return res;
1315  
1316  		XORMetric ourMetric;
1317  		if (closeThanUsOnly) ourMetric = destKey ^ i2p::context.GetIdentHash ();
1318  		for (auto& it: v)
1319  		{
1320  			if (closeThanUsOnly && ourMetric < (destKey ^ it->GetIdentHash ())) break;
1321  			res.push_back (it->GetIdentHash ());
1322  		}
1323  		return res;
1324  	}
1325  
1326  	std::shared_ptr<const RouterInfo> NetDb::GetRandomRouterInFamily (FamilyID fam) const
1327  	{
1328  		return GetRandomRouter(
1329  			[fam](std::shared_ptr<const RouterInfo> router)->bool
1330  		{
1331  			return router->IsFamily(fam);
1332  		});
1333  	}
1334  
1335  	std::vector<IdentHash> NetDb::GetExploratoryNonFloodfill (const IdentHash& destination,
1336  		size_t num, const std::unordered_set<IdentHash>& excluded)
1337  	{
1338  		std::vector<IdentHash> ret;
1339  		if (!num || m_RouterInfos.empty ()) return ret; // empty list
1340  		auto ts = i2p::util::GetMonotonicSeconds ();
1341  		if (ts > m_LastExploratorySelectionUpdateTime +	NETDB_EXPLORATORY_SELECTION_UPDATE_INTERVAL)
1342  		{
1343  			// update selection
1344  			m_ExploratorySelection.clear ();
1345  			std::vector<std::shared_ptr<const RouterInfo> > eligible;
1346  			eligible.reserve (m_RouterInfos.size ());
1347  			{
1348  				// collect eligible from current netdb
1349  				bool checkIsReal = i2p::tunnel::tunnels.GetPreciseTunnelCreationSuccessRate () < NETDB_TUNNEL_CREATION_RATE_THRESHOLD; // too low rate
1350  				std::lock_guard<std::mutex> l(m_RouterInfosMutex);
1351  				for (const auto& it: m_RouterInfos)
1352  					if (!it.second->IsDeclaredFloodfill () &&
1353  					 	(!checkIsReal || (it.second->HasProfile () && it.second->GetProfile ()->IsReal ())))
1354  							eligible.push_back (it.second);
1355  			}
1356  			if (eligible.size () > NETDB_MAX_EXPLORATORY_SELECTION_SIZE)
1357  			{
1358  				 std::sample (eligible.begin(), eligible.end(), std::back_inserter(m_ExploratorySelection),
1359  				 	NETDB_MAX_EXPLORATORY_SELECTION_SIZE, m_Rng);
1360  			}
1361  			else
1362  				std::swap (m_ExploratorySelection, eligible);
1363  			m_LastExploratorySelectionUpdateTime = ts;
1364  		}
1365  
1366  		// sort by distance
1367  		IdentHash destKey = CreateRoutingKey (destination);
1368  		std::map<XORMetric, std::shared_ptr<const RouterInfo> > sorted;
1369  		for (const auto& it: m_ExploratorySelection)
1370  			if (!excluded.count (it->GetIdentHash ()))
1371  				sorted.emplace (destKey ^ it->GetIdentHash (), it);
1372  		// return first num closest routers
1373  		for (const auto& it: sorted)
1374  		{
1375  			ret.push_back (it.second->GetIdentHash ());
1376  			if (ret.size () >= num) break;
1377  		}
1378  		return ret;
1379  	}
1380  
1381  	void NetDb::ManageRouterInfos ()
1382  	{
1383  		auto ts = i2p::util::GetSecondsSinceEpoch ();
1384  		{
1385  			std::lock_guard<std::mutex> l(m_RouterInfosMutex);
1386  			for (auto& it: m_RouterInfos)
1387  				it.second->UpdateIntroducers (ts);
1388  		}
1389  		SaveUpdated ();
1390  	}
1391  
1392  	void NetDb::ManageLeaseSets ()
1393  	{
1394  		auto ts = i2p::util::GetMillisecondsSinceEpoch ();
1395  		for (auto it = m_LeaseSets.begin (); it != m_LeaseSets.end ();)
1396  		{
1397  			if (!it->second->IsValid () || ts > it->second->GetExpirationTime () - LEASE_ENDDATE_THRESHOLD)
1398  			{
1399  				LogPrint (eLogInfo, "NetDb: LeaseSet ", it->first.ToBase64 (), " expired or invalid");
1400  				it = m_LeaseSets.erase (it);
1401  			}
1402  			else
1403  				++it;
1404  		}
1405  		m_LeasesPool.CleanUpMt ();
1406  	}
1407  
1408  	bool NetDb::PopulateRouterInfoBuffer (std::shared_ptr<RouterInfo> r)
1409  	{
1410  		if (!r) return false;
1411  		if (r->GetBuffer ()) return true;
1412  		return r->LoadBuffer (m_Storage.Path (r->GetIdentHashBase64 ()));
1413  	}
1414  }
1415  }