knownnodes.py
1 """ 2 Manipulations with knownNodes dictionary. 3 """ 4 # TODO: knownnodes object maybe? 5 # pylint: disable=global-statement 6 7 import json 8 import logging 9 import os 10 import pickle # nosec B403 11 import threading 12 import time 13 try: 14 from collections.abc import Iterable 15 except ImportError: 16 from collections import Iterable 17 18 import state 19 from bmconfigparser import config 20 from network.node import Peer 21 22 state.Peer = Peer 23 24 knownNodesLock = threading.RLock() 25 """Thread lock for knownnodes modification""" 26 knownNodes = {stream: {} for stream in range(1, 4)} 27 """The dict of known nodes for each stream""" 28 29 knownNodesTrimAmount = 2000 30 """trim stream knownnodes dict to this length""" 31 32 knownNodesForgetRating = -0.5 33 """forget a node after rating is this low""" 34 35 knownNodesActual = False 36 37 logger = logging.getLogger('default') 38 39 DEFAULT_NODES = ( 40 Peer('5.45.99.75', 8444), 41 Peer('75.167.159.54', 8444), 42 Peer('95.165.168.168', 8444), 43 Peer('85.180.139.241', 8444), 44 Peer('158.222.217.190', 8080), 45 Peer('178.62.12.187', 8448), 46 Peer('24.188.198.204', 8111), 47 Peer('109.147.204.113', 1195), 48 Peer('178.11.46.221', 8444) 49 ) 50 51 52 def json_serialize_knownnodes(output): 53 """ 54 Reorganize knownnodes dict and write it as JSON to output 55 """ 56 _serialized = [] 57 for stream, peers in knownNodes.items(): 58 for peer, info in peers.items(): 59 info.update(rating=round(info.get('rating', 0), 2)) 60 _serialized.append({ 61 'stream': stream, 'peer': peer._asdict(), 'info': info 62 }) 63 json.dump(_serialized, output, indent=4) 64 65 66 def json_deserialize_knownnodes(source): 67 """ 68 Read JSON from source and make knownnodes dict 69 """ 70 global knownNodesActual 71 for node in json.load(source): 72 peer = node['peer'] 73 info = node['info'] 74 peer = Peer(str(peer['host']), peer.get('port', 8444)) 75 knownNodes[node['stream']][peer] = info 76 if not (knownNodesActual 77 or info.get('self')) and peer not in DEFAULT_NODES: 78 knownNodesActual = True 79 80 81 def pickle_deserialize_old_knownnodes(source): 82 """ 83 Unpickle source and reorganize knownnodes dict if it has old format 84 the old format was {Peer:lastseen, ...} 85 the new format is {Peer:{"lastseen":i, "rating":f}} 86 """ 87 global knownNodes 88 knownNodes = pickle.load(source) # nosec B301 89 for stream in list(knownNodes.keys()): 90 for node, params in knownNodes[stream].items(): 91 if isinstance(params, (float, int)): 92 addKnownNode(stream, node, params) 93 94 95 def saveKnownNodes(dirName=None): 96 """Save knownnodes to filesystem""" 97 if dirName is None: 98 dirName = state.appdata 99 with knownNodesLock: 100 with open(os.path.join(dirName, 'knownnodes.dat'), 'wb') as output: 101 json_serialize_knownnodes(output) 102 103 104 def addKnownNode(stream, peer, lastseen=None, is_self=False): 105 """ 106 Add a new node to the dict or update lastseen if it already exists. 107 Do it for each stream number if *stream* is `Iterable`. 108 Returns True if added a new node. 109 """ 110 # pylint: disable=too-many-branches 111 if isinstance(stream, Iterable): 112 with knownNodesLock: 113 for s in stream: 114 addKnownNode(s, peer, lastseen, is_self) 115 return 116 117 rating = 0.0 118 if not lastseen: 119 # FIXME: maybe about 28 days? 120 lastseen = int(time.time()) 121 else: 122 lastseen = int(lastseen) 123 try: 124 info = knownNodes[stream].get(peer) 125 if lastseen > info['lastseen']: 126 info['lastseen'] = lastseen 127 except (KeyError, TypeError): 128 pass 129 else: 130 return 131 132 if not is_self: 133 if len(knownNodes[stream]) > config.safeGetInt( 134 "knownnodes", "maxnodes"): 135 return 136 137 knownNodes[stream][peer] = { 138 'lastseen': lastseen, 139 'rating': rating or 1 if is_self else 0, 140 'self': is_self, 141 } 142 return True 143 144 145 def createDefaultKnownNodes(): 146 """Creating default Knownnodes""" 147 past = time.time() - 2418600 # 28 days - 10 min 148 for peer in DEFAULT_NODES: 149 addKnownNode(1, peer, past) 150 saveKnownNodes() 151 152 153 def readKnownNodes(): 154 """Load knownnodes from filesystem""" 155 try: 156 with open(state.appdata + 'knownnodes.dat', 'rb') as source: 157 with knownNodesLock: 158 try: 159 json_deserialize_knownnodes(source) 160 except ValueError: 161 source.seek(0) 162 pickle_deserialize_old_knownnodes(source) 163 except (IOError, OSError, KeyError, EOFError): 164 logger.debug( 165 'Failed to read nodes from knownnodes.dat', exc_info=True) 166 createDefaultKnownNodes() 167 168 # your own onion address, if setup 169 onionhostname = config.safeGet('bitmessagesettings', 'onionhostname') 170 if onionhostname and ".onion" in onionhostname: 171 onionport = config.safeGetInt('bitmessagesettings', 'onionport') 172 if onionport: 173 self_peer = Peer(onionhostname, onionport) 174 addKnownNode(1, self_peer, is_self=True) 175 state.ownAddresses[self_peer] = True 176 177 178 def increaseRating(peer): 179 """Increase rating of a peer node""" 180 increaseAmount = 0.1 181 maxRating = 1 182 with knownNodesLock: 183 for stream in list(knownNodes.keys()): 184 try: 185 knownNodes[stream][peer]["rating"] = min( 186 knownNodes[stream][peer]["rating"] + increaseAmount, 187 maxRating 188 ) 189 except KeyError: 190 pass 191 192 193 def decreaseRating(peer): 194 """Decrease rating of a peer node""" 195 decreaseAmount = 0.1 196 minRating = -1 197 with knownNodesLock: 198 for stream in list(knownNodes.keys()): 199 try: 200 knownNodes[stream][peer]["rating"] = max( 201 knownNodes[stream][peer]["rating"] - decreaseAmount, 202 minRating 203 ) 204 except KeyError: 205 pass 206 207 208 def trimKnownNodes(recAddrStream=1): 209 """Triming Knownnodes""" 210 if len(knownNodes[recAddrStream]) < \ 211 config.safeGetInt("knownnodes", "maxnodes"): 212 return 213 with knownNodesLock: 214 oldestList = sorted( 215 knownNodes[recAddrStream], 216 key=lambda x: x['lastseen'] 217 )[:knownNodesTrimAmount] 218 for oldest in oldestList: 219 del knownNodes[recAddrStream][oldest] 220 221 222 def dns(): 223 """Add DNS names to knownnodes""" 224 for port in [8080, 8444]: 225 addKnownNode( 226 1, Peer('bootstrap%s.bitmessage.org' % port, port)) 227 228 229 def cleanupKnownNodes(pool): 230 """ 231 Cleanup knownnodes: remove old nodes and nodes with low rating 232 """ 233 global knownNodesActual 234 now = int(time.time()) 235 needToWriteKnownNodesToDisk = False 236 237 with knownNodesLock: 238 for stream in knownNodes: 239 if stream not in pool.streams: 240 continue 241 keys = list(knownNodes[stream].keys()) 242 for node in keys: 243 if len(knownNodes[stream]) <= 1: # leave at least one node 244 if stream == 1: 245 knownNodesActual = False 246 break 247 try: 248 age = now - knownNodes[stream][node]["lastseen"] 249 # scrap old nodes (age > 28 days) 250 if age > 2419200: 251 needToWriteKnownNodesToDisk = True 252 del knownNodes[stream][node] 253 continue 254 # scrap old nodes (age > 3 hours) with low rating 255 if (age > 10800 and knownNodes[stream][node]["rating"] 256 <= knownNodesForgetRating): 257 needToWriteKnownNodesToDisk = True 258 del knownNodes[stream][node] 259 continue 260 except TypeError: 261 logger.warning('Error in %s', node) 262 keys = [] 263 264 # Let us write out the knowNodes to disk 265 # if there is anything new to write out. 266 if needToWriteKnownNodesToDisk: 267 saveKnownNodes()