1  """
  2  Low-level protocol-related functions.
  3  """
  4  # pylint: disable=too-many-boolean-expressions,too-many-return-statements
  5  # pylint: disable=too-many-locals,too-many-statements
  6  
  7  import base64
  8  import hashlib
  9  import random
 10  import socket
 11  import sys
 12  import time
 13  from binascii import hexlify
 14  from struct import Struct, pack, unpack
 15  
 16  import defaults
 17  import highlevelcrypto
 18  import state
 19  from addresses import (decodeAddress, decodeVarint, encodeVarint,
 20                         varintDecodeError)
 21  from bmconfigparser import config
 22  from debug import logger
 23  from helper_sql import sqlExecute
 24  from network.node import Peer
 25  from version import softwareVersion
 26  
 27  # Network constants
 28  magic = 0xE9BEB4D9
 29  #: protocol specification says max 1000 addresses in one addr command
 30  MAX_ADDR_COUNT = 1000
 31  #: address is online if online less than this many seconds ago
 32  ADDRESS_ALIVE = 10800
 33  #: ~1.6 MB which is the maximum possible size of an inv message.
 34  MAX_MESSAGE_SIZE = 1600100
 35  #: 2**18 = 256kB is the maximum size of an object payload
 36  MAX_OBJECT_PAYLOAD_SIZE = 2**18
 37  #: protocol specification says max 50000 objects in one inv command
 38  MAX_OBJECT_COUNT = 50000
 39  #: maximum time offset
 40  MAX_TIME_OFFSET = 3600
 41  
 42  # Service flags
 43  #: This is a normal network node
 44  NODE_NETWORK = 1
 45  #: This node supports SSL/TLS in the current connect (python < 2.7.9
 46  #: only supports an SSL client, so in that case it would only have this
 47  #: on when the connection is a client).
 48  NODE_SSL = 2
 49  # (Proposal) This node may do PoW on behalf of some its peers
 50  # (PoW offloading/delegating), but it doesn't have to. Clients may have
 51  # to meet additional requirements (e.g. TLS authentication)
 52  # NODE_POW = 4
 53  #: Node supports dandelion
 54  NODE_DANDELION = 8
 55  
 56  # Bitfield flags
 57  BITFIELD_DOESACK = 1
 58  
 59  # Error types
 60  STATUS_WARNING = 0
 61  STATUS_ERROR = 1
 62  STATUS_FATAL = 2
 63  
 64  # Object types
 65  OBJECT_GETPUBKEY = 0
 66  OBJECT_PUBKEY = 1
 67  OBJECT_MSG = 2
 68  OBJECT_BROADCAST = 3
 69  OBJECT_ONIONPEER = 0x746f72
 70  OBJECT_I2P = 0x493250
 71  OBJECT_ADDR = 0x61646472
 72  
 73  eightBytesOfRandomDataUsedToDetectConnectionsToSelf = pack(
 74      '>Q', random.randrange(1, 18446744073709551615))  # nosec B311
 75  
 76  # Compiled struct for packing/unpacking headers
 77  # New code should use CreatePacket instead of Header.pack
 78  Header = Struct('!L12sL4s')
 79  
 80  VersionPacket = Struct('>LqQ20s4s36sH')
 81  
 82  # Bitfield
 83  
 84  
 85  def getBitfield(address):
 86      """Get a bitfield from an address"""
 87      # bitfield of features supported by me (see the wiki).
 88      bitfield = 0
 89      # send ack
 90      if not config.safeGetBoolean(address, 'dontsendack'):
 91          bitfield |= BITFIELD_DOESACK
 92      return pack('>I', bitfield)
 93  
 94  
 95  def checkBitfield(bitfieldBinary, flags):
 96      """Check if a bitfield matches the given flags"""
 97      bitfield, = unpack('>I', bitfieldBinary)
 98      return (bitfield & flags) == flags
 99  
100  
101  def isBitSetWithinBitfield(fourByteString, n):
102      """Check if a particular bit is set in a bitfeld"""
103      # Uses MSB 0 bit numbering across 4 bytes of data
104      n = 31 - n
105      x, = unpack('>L', fourByteString)
106      return x & 2**n != 0
107  
108  # Streams
109  
110  
111  MIN_VALID_STREAM = 1
112  MAX_VALID_STREAM = 2**63 - 1
113  
114  # IP addresses
115  
116  
117  def encodeHost(host):
118      """Encode a given host to be used in low-level socket operations"""
119      if host.endswith('.onion'):
120          return b'\xfd\x87\xd8\x7e\xeb\x43' + base64.b32decode(
121              host.split(".")[0], True)
122      elif host.find(':') == -1:
123          return b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + \
124              socket.inet_aton(host)
125      return socket.inet_pton(socket.AF_INET6, host)
126  
127  
128  def networkType(host):
129      """Determine if a host is IPv4, IPv6 or an onion address"""
130      if host.endswith('.onion'):
131          return 'onion'
132      elif host.find(':') == -1:
133          return 'IPv4'
134      return 'IPv6'
135  
136  
137  def network_group(host):
138      """Canonical identifier of network group
139         simplified, borrowed from
140         GetGroup() in src/netaddresses.cpp in bitcoin core"""
141      if not isinstance(host, str):
142          return None
143      network_type = networkType(host)
144      try:
145          raw_host = encodeHost(host)
146      except socket.error:
147          return host
148      if network_type == 'IPv4':
149          decoded_host = checkIPv4Address(raw_host[12:], True)
150          if decoded_host:
151              # /16 subnet
152              return raw_host[12:14]
153      elif network_type == 'IPv6':
154          decoded_host = checkIPv6Address(raw_host, True)
155          if decoded_host:
156              # /32 subnet
157              return raw_host[0:12]
158      else:
159          # just host, e.g. for tor
160          return host
161      # global network type group for local, private, unroutable
162      return network_type
163  
164  
165  def checkIPAddress(host, private=False):
166      """
167      Returns hostStandardFormat if it is a valid IP address,
168      otherwise returns False
169      """
170      if host[0:12] == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF':
171          hostStandardFormat = socket.inet_ntop(socket.AF_INET, host[12:])
172          return checkIPv4Address(host[12:], hostStandardFormat, private)
173      elif host[0:6] == b'\xfd\x87\xd8\x7e\xeb\x43':
174          # Onion, based on BMD/bitcoind
175          hostStandardFormat = base64.b32encode(host[6:]).lower() + ".onion"
176          if private:
177              return False
178          return hostStandardFormat
179      else:
180          try:
181              hostStandardFormat = socket.inet_ntop(socket.AF_INET6, host)
182          except ValueError:
183              return False
184          if len(hostStandardFormat) == 0:
185              # This can happen on Windows systems which are
186              # not 64-bit compatible so let us drop the IPv6 address.
187              return False
188          return checkIPv6Address(host, hostStandardFormat, private)
189  
190  
191  def checkIPv4Address(host, hostStandardFormat, private=False):
192      """
193      Returns hostStandardFormat if it is an IPv4 address,
194      otherwise returns False
195      """
196      if host[0:1] == b'\x7F':  # 127/8
197          if not private:
198              logger.debug(
199                  'Ignoring IP address in loopback range: %s',
200                  hostStandardFormat)
201          return hostStandardFormat if private else False
202      if host[0:1] == b'\x0A':  # 10/8
203          if not private:
204              logger.debug(
205                  'Ignoring IP address in private range: %s', hostStandardFormat)
206          return hostStandardFormat if private else False
207      if host[0:2] == b'\xC0\xA8':  # 192.168/16
208          if not private:
209              logger.debug(
210                  'Ignoring IP address in private range: %s', hostStandardFormat)
211          return hostStandardFormat if private else False
212      if host[0:2] >= b'\xAC\x10' and host[0:2] < b'\xAC\x20':  # 172.16/12
213          if not private:
214              logger.debug(
215                  'Ignoring IP address in private range: %s', hostStandardFormat)
216          return hostStandardFormat if private else False
217      return False if private else hostStandardFormat
218  
219  
220  def checkIPv6Address(host, hostStandardFormat, private=False):
221      """
222      Returns hostStandardFormat if it is an IPv6 address,
223      otherwise returns False
224      """
225      if host == b'\x00' * 15 + b'\x01':
226          if not private:
227              logger.debug('Ignoring loopback address: %s', hostStandardFormat)
228          return False
229      try:
230          host = [ord(c) for c in host[:2]]
231      except TypeError:  # python3 has ints already
232          pass
233      if host[0] == 0xfe and host[1] & 0xc0 == 0x80:
234          if not private:
235              logger.debug('Ignoring local address: %s', hostStandardFormat)
236          return hostStandardFormat if private else False
237      if host[0] & 0xfe == 0xfc:
238          if not private:
239              logger.debug(
240                  'Ignoring unique local address: %s', hostStandardFormat)
241          return hostStandardFormat if private else False
242      return False if private else hostStandardFormat
243  
244  
245  def haveSSL(server=False):
246      """
247      Predicate to check if ECDSA server support is required and available
248  
249      python < 2.7.9's ssl library does not support ECDSA server due to
250      missing initialisation of available curves, but client works ok
251      """
252      if not server:
253          return True
254      elif sys.version_info >= (2, 7, 9):
255          return True
256      return False
257  
258  
259  def checkSocksIP(host):
260      """Predicate to check if we're using a SOCKS proxy"""
261      sockshostname = config.safeGet(
262          'bitmessagesettings', 'sockshostname')
263      try:
264          if not state.socksIP:
265              state.socksIP = socket.gethostbyname(sockshostname)
266      except NameError:  # uninitialised
267          state.socksIP = socket.gethostbyname(sockshostname)
268      except (TypeError, socket.gaierror):  # None, resolving failure
269          state.socksIP = sockshostname
270      return state.socksIP == host
271  
272  
273  def isProofOfWorkSufficient(
274          data, nonceTrialsPerByte=0, payloadLengthExtraBytes=0, recvTime=0):
275      """
276      Validate an object's Proof of Work using method described
277      :doc:`here </pow>`
278  
279      Arguments:
280          int nonceTrialsPerByte (default: from `.defaults`)
281          int payloadLengthExtraBytes (default: from `.defaults`)
282          float recvTime (optional) UNIX epoch time when object was
283            received from the network (default: current system time)
284      Returns:
285          True if PoW valid and sufficient, False in all other cases
286      """
287      if nonceTrialsPerByte < defaults.networkDefaultProofOfWorkNonceTrialsPerByte:
288          nonceTrialsPerByte = defaults.networkDefaultProofOfWorkNonceTrialsPerByte
289      if payloadLengthExtraBytes < defaults.networkDefaultPayloadLengthExtraBytes:
290          payloadLengthExtraBytes = defaults.networkDefaultPayloadLengthExtraBytes
291      endOfLifeTime, = unpack('>Q', data[8:16])
292      TTL = endOfLifeTime - int(recvTime if recvTime else time.time())
293      if TTL < 300:
294          TTL = 300
295      POW, = unpack('>Q', highlevelcrypto.double_sha512(
296          data[:8] + hashlib.sha512(data[8:]).digest())[0:8])
297      return POW <= 2 ** 64 / (
298          nonceTrialsPerByte * (
299              len(data) + payloadLengthExtraBytes
300              + ((TTL * (len(data) + payloadLengthExtraBytes)) / (2 ** 16))))
301  
302  
303  # Packet creation
304  
305  
306  def CreatePacket(command, payload=b''):
307      """Construct and return a packet"""
308      payload_length = len(payload)
309      checksum = hashlib.sha512(payload).digest()[0:4]
310  
311      b = bytearray(Header.size + payload_length)
312      Header.pack_into(b, 0, magic, command, payload_length, checksum)
313      b[Header.size:] = payload
314      return bytes(b)
315  
316  
317  def assembleAddrMessage(peerList):
318      """Create address command"""
319      if isinstance(peerList, Peer):
320          peerList = [peerList]
321      if not peerList:
322          return b''
323      retval = b''
324      for i in range(0, len(peerList), MAX_ADDR_COUNT):
325          payload = encodeVarint(len(peerList[i:i + MAX_ADDR_COUNT]))
326          for stream, peer, timestamp in peerList[i:i + MAX_ADDR_COUNT]:
327              # 64-bit time
328              payload += pack('>Q', timestamp)
329              payload += pack('>I', stream)
330              # service bit flags offered by this node
331              payload += pack('>q', 1)
332              payload += encodeHost(peer.host)
333              # remote port
334              payload += pack('>H', peer.port)
335          retval += CreatePacket(b'addr', payload)
336      return retval
337  
338  
339  def assembleVersionMessage(
340      remoteHost, remotePort, participatingStreams,
341      dandelion_enabled=True, server=False, nodeid=None
342  ):
343      """
344      Construct the payload of a version message,
345      return the resulting bytes of running `CreatePacket` on it
346      """
347      payload = b''
348      payload += pack('>L', 3)  # protocol version.
349      # bitflags of the services I offer.
350      payload += pack(
351          '>q',
352          NODE_NETWORK
353          | (NODE_SSL if haveSSL(server) else 0)
354          | (NODE_DANDELION if dandelion_enabled else 0)
355      )
356      payload += pack('>q', int(time.time()))
357  
358      # boolservices of remote connection; ignored by the remote host.
359      payload += pack('>q', 1)
360      if checkSocksIP(remoteHost) and server:
361          # prevent leaking of tor outbound IP
362          payload += encodeHost('127.0.0.1')
363          payload += pack('>H', 8444)
364      else:
365          # use first 16 bytes if host data is longer
366          # for example in case of onion v3 service
367          try:
368              payload += encodeHost(remoteHost)[:16]
369          except socket.error:
370              payload += encodeHost('127.0.0.1')
371          payload += pack('>H', remotePort)  # remote IPv6 and port
372  
373      # bitflags of the services I offer.
374      payload += pack(
375          '>q',
376          NODE_NETWORK
377          | (NODE_SSL if haveSSL(server) else 0)
378          | (NODE_DANDELION if dandelion_enabled else 0)
379      )
380      # = 127.0.0.1. This will be ignored by the remote host.
381      # The actual remote connected IP will be used.
382      payload += b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF' + pack(
383          '>L', 2130706433)
384      # we have a separate extPort and incoming over clearnet
385      # or outgoing through clearnet
386      extport = config.safeGetInt('bitmessagesettings', 'extport')
387      if (
388          extport and ((server and not checkSocksIP(remoteHost)) or (
389              config.get('bitmessagesettings', 'socksproxytype')
390              == 'none' and not server))
391      ):
392          payload += pack('>H', extport)
393      elif checkSocksIP(remoteHost) and server:  # incoming connection over Tor
394          payload += pack(
395              '>H', config.getint('bitmessagesettings', 'onionport'))
396      else:  # no extport and not incoming over Tor
397          payload += pack(
398              '>H', config.getint('bitmessagesettings', 'port'))
399  
400      if nodeid is not None:
401          payload += nodeid[0:8]
402      else:
403          payload += eightBytesOfRandomDataUsedToDetectConnectionsToSelf
404      userAgent = ('/PyBitmessage:%s/' % softwareVersion).encode('utf-8')
405      payload += encodeVarint(len(userAgent))
406      payload += userAgent
407  
408      # Streams
409      payload += encodeVarint(len(participatingStreams))
410      count = 0
411      for stream in sorted(participatingStreams):
412          payload += encodeVarint(stream)
413          count += 1
414          # protocol limit, see specification
415          if count >= 160000:
416              break
417  
418      return CreatePacket(b'version', payload)
419  
420  
421  def assembleErrorMessage(fatal=0, banTime=0, inventoryVector='', errorText=''):
422      """
423      Construct the payload of an error message,
424      return the resulting bytes of running `CreatePacket` on it
425      """
426      payload = encodeVarint(fatal)
427      payload += encodeVarint(banTime)
428      payload += encodeVarint(len(inventoryVector))
429      payload += inventoryVector
430      payload += encodeVarint(len(errorText))
431      payload += errorText
432      return CreatePacket(b'error', payload)
433  
434  
435  # Packet decoding
436  
437  
438  def decodeObjectParameters(data):
439      """Decode the parameters of a raw object needed to put it in inventory"""
440      # BMProto.decode_payload_content("QQIvv")
441      expiresTime = unpack('>Q', data[8:16])[0]
442      objectType = unpack('>I', data[16:20])[0]
443      parserPos = 20 + decodeVarint(data[20:30])[1]
444      toStreamNumber = decodeVarint(data[parserPos:parserPos + 10])[0]
445  
446      return objectType, toStreamNumber, expiresTime
447  
448  
449  def decryptAndCheckPubkeyPayload(data, address):
450      """
451      Version 4 pubkeys are encrypted. This function is run when we
452      already have the address to which we want to try to send a message.
453      The 'data' may come either off of the wire or we might have had it
454      already in our inventory when we tried to send a msg to this
455      particular address.
456      """
457      try:
458          addressVersion, streamNumber, ripe = decodeAddress(address)[1:]
459  
460          readPosition = 20  # bypass the nonce, time, and object type
461          embeddedAddressVersion, varintLength = decodeVarint(
462              data[readPosition:readPosition + 10])
463          readPosition += varintLength
464          embeddedStreamNumber, varintLength = decodeVarint(
465              data[readPosition:readPosition + 10])
466          readPosition += varintLength
467          # We'll store the address version and stream number
468          # (and some more) in the pubkeys table.
469          storedData = data[20:readPosition]
470  
471          if addressVersion != embeddedAddressVersion:
472              logger.info(
473                  'Pubkey decryption was UNsuccessful'
474                  ' due to address version mismatch.')
475              return 'failed'
476          if streamNumber != embeddedStreamNumber:
477              logger.info(
478                  'Pubkey decryption was UNsuccessful'
479                  ' due to stream number mismatch.')
480              return 'failed'
481  
482          tag = data[readPosition:readPosition + 32]
483          readPosition += 32
484          # the time through the tag. More data is appended onto
485          # signedData below after the decryption.
486          signedData = data[8:readPosition]
487          encryptedData = data[readPosition:]
488  
489          # Let us try to decrypt the pubkey
490          toAddress, cryptorObject = state.neededPubkeys[tag]
491          if toAddress != address:
492              logger.critical(
493                  'decryptAndCheckPubkeyPayload failed due to toAddress'
494                  ' mismatch. This is very peculiar.'
495                  ' toAddress: %s, address %s',
496                  toAddress, address
497              )
498              # the only way I can think that this could happen
499              # is if someone encodes their address data two different ways.
500              # That sort of address-malleability should have been caught
501              # by the UI or API and an error given to the user.
502              return 'failed'
503          try:
504              decryptedData = cryptorObject.decrypt(encryptedData)
505          except:  # noqa:E722
506              # FIXME: use a proper exception after `pyelliptic.ecc` is refactored.
507              # Someone must have encrypted some data with a different key
508              # but tagged it with a tag for which we are watching.
509              logger.info('Pubkey decryption was unsuccessful.')
510              return 'failed'
511  
512          readPosition = 0
513          # bitfieldBehaviors = decryptedData[readPosition:readPosition + 4]
514          readPosition += 4
515          pubSigningKey = '\x04' + decryptedData[readPosition:readPosition + 64]
516          readPosition += 64
517          pubEncryptionKey = '\x04' + decryptedData[readPosition:readPosition + 64]
518          readPosition += 64
519          specifiedNonceTrialsPerByteLength = decodeVarint(
520              decryptedData[readPosition:readPosition + 10])[1]
521          readPosition += specifiedNonceTrialsPerByteLength
522          specifiedPayloadLengthExtraBytesLength = decodeVarint(
523              decryptedData[readPosition:readPosition + 10])[1]
524          readPosition += specifiedPayloadLengthExtraBytesLength
525          storedData += decryptedData[:readPosition]
526          signedData += decryptedData[:readPosition]
527          signatureLength, signatureLengthLength = decodeVarint(
528              decryptedData[readPosition:readPosition + 10])
529          readPosition += signatureLengthLength
530          signature = decryptedData[readPosition:readPosition + signatureLength]
531  
532          if not highlevelcrypto.verify(
533                  signedData, signature, hexlify(pubSigningKey)):
534              logger.info(
535                  'ECDSA verify failed (within decryptAndCheckPubkeyPayload)')
536              return 'failed'
537  
538          logger.info(
539              'ECDSA verify passed (within decryptAndCheckPubkeyPayload)')
540  
541          embeddedRipe = highlevelcrypto.to_ripe(pubSigningKey, pubEncryptionKey)
542  
543          if embeddedRipe != ripe:
544              # Although this pubkey object had the tag were were looking for
545              # and was encrypted with the correct encryption key,
546              # it doesn't contain the correct pubkeys. Someone is
547              # either being malicious or using buggy software.
548              logger.info(
549                  'Pubkey decryption was UNsuccessful due to RIPE mismatch.')
550              return 'failed'
551  
552          # Everything checked out. Insert it into the pubkeys table.
553  
554          logger.info(
555              'within decryptAndCheckPubkeyPayload, '
556              'addressVersion: %s, streamNumber: %s\nripe %s\n'
557              'publicSigningKey in hex: %s\npublicEncryptionKey in hex: %s',
558              addressVersion, streamNumber, hexlify(ripe),
559              hexlify(pubSigningKey), hexlify(pubEncryptionKey)
560          )
561  
562          t = (address, addressVersion, storedData, int(time.time()), 'yes')
563          sqlExecute('''INSERT INTO pubkeys VALUES (?,?,?,?,?)''', *t)
564          return 'successful'
565      except varintDecodeError:
566          logger.info(
567              'Pubkey decryption was UNsuccessful due to a malformed varint.')
568          return 'failed'
569      except Exception:
570          logger.critical(
571              'Pubkey decryption was UNsuccessful because of'
572              ' an unhandled exception! This is definitely a bug!',
573              exc_info=True
574          )
575          return 'failed'