From 19d9b1a4a507ef501eda10cdf2a7c9d3e200eed9 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Tue, 17 Apr 2018 17:46:48 +0200 Subject: [PATCH] Proof handling --- .gitignore | 3 +- RNS/Destination.py | 6 +- RNS/Identity.py | 26 +++-- RNS/Interfaces/AX25KISSInterface.py | 1 - RNS/Packet.py | 135 +++++++++++++++++++++--- RNS/Reticulum.py | 14 ++- RNS/Transport.py | 153 ++++++++++++++++++++++------ RNS/__init__.py | 1 + RNS/vendor/configobj.py | 2 +- 9 files changed, 283 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index de4e3f9..7d09436 100755 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .DS_Store *.pyc -t.py -t2.py +testutils TODO diff --git a/RNS/Destination.py b/RNS/Destination.py index 9d6e252..8d0f063 100755 --- a/RNS/Destination.py +++ b/RNS/Destination.py @@ -13,7 +13,7 @@ class Callbacks: def __init__(self): self.link_established = None self.packet = None - self.proof = None + self.proof_requested = None class Destination: KEYSIZE = RNS.Identity.KEYSIZE; @@ -101,8 +101,8 @@ class Destination: def packet_callback(self, callback): self.callbacks.packet = callback - def proof_callback(self, callback): - self.callbacks.proof = callback + def proof_requested_callback(self, callback): + self.callbacks.proof_requested = callback def setProofStrategy(self, proof_strategy): if not proof_strategy in Destination.proof_strategies: diff --git a/RNS/Identity.py b/RNS/Identity.py index c652aa9..e802b2b 100644 --- a/RNS/Identity.py +++ b/RNS/Identity.py @@ -14,12 +14,14 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric import padding class Identity: - #KEYSIZE = 1536 - KEYSIZE = 1024 - DERKEYSIZE = KEYSIZE+272 + #KEYSIZE = 1536 + KEYSIZE = 1024 + DERKEYSIZE = KEYSIZE+272 - # Padding size, not configurable - PADDINGSIZE= 336 + # Non-configurable constants + PADDINGSIZE = 336 # In bits + HASHLENGTH = 256 # In bits + SIGLENGTH = KEYSIZE # Storage known_destinations = {} @@ -257,13 +259,21 @@ class Identity: hashes.SHA256() ) return True - except: + except Exception as e: return False else: raise KeyError("Signature validation failed because identity does not hold a public key") - def prove(self, packet, destination): - proof_data = packet.packet_hash + self.sign(packet.packet_hash) + def prove(self, packet, destination=None): + signature = self.sign(packet.packet_hash) + if RNS.Reticulum.should_use_implicit_proof(): + proof_data = signature + else: + proof_data = packet.packet_hash + signature + + if destination == None: + destination = packet.generateProofDestination() + proof = RNS.Packet(destination, proof_data, RNS.Packet.PROOF) proof.send() diff --git a/RNS/Interfaces/AX25KISSInterface.py b/RNS/Interfaces/AX25KISSInterface.py index 6c850e0..47cc713 100644 --- a/RNS/Interfaces/AX25KISSInterface.py +++ b/RNS/Interfaces/AX25KISSInterface.py @@ -28,7 +28,6 @@ class AX25(): CRC_CORRECT = chr(0xF0)+chr(0xB8) -# TODO: THIS CLASS IS NOT YET IMPLEMENTED --- PLACEHOLDER ONLY --- class AX25KISSInterface(Interface): MAX_CHUNK = 32768 diff --git a/RNS/Packet.py b/RNS/Packet.py index e94e731..38e7fae 100755 --- a/RNS/Packet.py +++ b/RNS/Packet.py @@ -2,6 +2,96 @@ import struct import time import RNS +class PacketReceipt: + # Receipt status constants + FAILED = 0x00 + SENT = 0x01 + DELIVERED = 0x02 + + + EXPL_LENGTH = RNS.Identity.HASHLENGTH/8+RNS.Identity.SIGLENGTH/8 + IMPL_LENGTH = RNS.Identity.SIGLENGTH/8 + + # Creates a new packet receipt from a sent packet + def __init__(self, packet): + self.hash = packet.getHash() + self.sent = True + self.sent_at = time.time() + self.timeout = Packet.TIMEOUT + self.proved = False + self.status = PacketReceipt.SENT + self.destination = packet.destination + self.callbacks = PacketReceiptCallbacks() + self.concluded_at = None + + # Validate a proof packet + def validateProofPacket(self, proof_packet): + return self.validateProof(proof_packet.data) + + # Validate a raw proof + def validateProof(self, proof): + if len(proof) == PacketReceipt.EXPL_LENGTH: + # This is an explicit proof + proof_hash = proof[:RNS.Identity.HASHLENGTH/8] + signature = proof[RNS.Identity.HASHLENGTH/8:RNS.Identity.HASHLENGTH/8+RNS.Identity.SIGLENGTH/8] + if proof_hash == self.hash: + proof_valid = self.destination.identity.validate(signature, self.hash) + if proof_valid: + self.status = PacketReceipt.DELIVERED + self.proved = True + if self.callbacks.delivery != None: + self.callbacks.delivery(self) + return True + else: + return False + else: + return False + elif len(proof) == PacketReceipt.IMPL_LENGTH: + # This is an implicit proof + signature = proof[:RNS.Identity.SIGLENGTH/8] + proof_valid = self.destination.identity.validate(signature, self.hash) + if proof_valid: + self.status = PacketReceipt.DELIVERED + self.proved = True + if self.callbacks.delivery != None: + self.callbacks.delivery(self) + return True + else: + return False + else: + return False + + + def isTimedOut(self): + return (self.sent_at+self.timeout < time.time()) + + def checkTimeout(self): + if self.isTimedOut(): + self.status = PacketReceipt.FAILED + self.concluded_at = time.time() + if self.callbacks.timeout: + self.callbacks.timeout(self) + + + # Set the timeout in seconds + def setTimeout(self, timeout): + self.timeout = float(timeout) + + # Set a function that gets called when + # a successfull delivery has been proved + def delivery_callback(self, callback): + self.callbacks.delivery = callback + + # Set a function that gets called if the + # delivery times out + def timeout_callback(self, callback): + self.callbacks.timeout = callback + +class PacketReceiptCallbacks: + def __init__(self): + self.delivery = None + self.timeout = None + class Packet: # Constants DATA = 0x00; @@ -16,6 +106,9 @@ class Packet: HEADER_4 = 0x03; # Reserved header_types = [HEADER_1, HEADER_2, HEADER_3, HEADER_4] + # Defaults + TIMEOUT = 3600.0 + def __init__(self, destination, data, packet_type = DATA, transport_type = RNS.Transport.BROADCAST, header_type = HEADER_1, transport_id = None): if destination != None: if transport_type == None: @@ -35,6 +128,7 @@ class Packet: self.raw = None self.packed = False self.sent = False + self.receipt = None self.fromPacked = False else: self.raw = data @@ -67,6 +161,7 @@ class Packet: self.ciphertext = self.destination.encrypt(self.data) else: self.ciphertext = self.data + if self.header_type == Packet.HEADER_3: self.header += self.destination.link_id self.ciphertext = self.data @@ -103,10 +198,11 @@ class Packet: if not self.packed: self.pack() - RNS.Transport.outbound(self) - self.packet_hash = RNS.Identity.fullHash(self.raw) - self.sent_at = time.time() - self.sent = True + if RNS.Transport.outbound(self): + return self.receipt + else: + # TODO: Don't raise error here, handle gracefully + raise IOError("Packet could not be sent! Do you have any outbound interfaces configured?") else: raise IOError("Packet was already sent") @@ -116,21 +212,36 @@ class Packet: else: raise IOError("Packet was not sent yet") - def prove(self, destination): + def prove(self, destination=None): if self.fromPacked and self.destination: if self.destination.identity and self.destination.identity.prv: self.destination.identity.prove(self, destination) + # Generates a special destination that allows Reticulum + # to direct the proof back to the proved packet's sender + def generateProofDestination(self): + return ProofDestination(self) + def validateProofPacket(self, proof_packet): - return self.validateProof(proof_packet.data) + return self.receipt.validateProofPacket(proof_packet) def validateProof(self, proof): - proof_hash = proof[:32] - signature = proof[32:] - if proof_hash == self.packet_hash: - return self.destination.identity.validate(signature, proof_hash) - else: - return False + return self.receipt.validateProof(proof) + def updateHash(self): + self.packet_hash = self.getHash() + def getHash(self): + return RNS.Identity.fullHash(self.getHashablePart()) + + def getHashablePart(self): + return self.raw[0:1]+self.raw[2:] + +class ProofDestination: + def __init__(self, packet): + self.hash = packet.getHash()[:10]; + self.type = RNS.Destination.SINGLE + + def encrypt(self, plaintext): + return plaintext diff --git a/RNS/Reticulum.py b/RNS/Reticulum.py index 5069f65..e078053 100755 --- a/RNS/Reticulum.py +++ b/RNS/Reticulum.py @@ -29,6 +29,7 @@ class Reticulum: Reticulum.cachepath = Reticulum.configdir+"/storage/cache" Reticulum.__allow_unencrypted = False + Reticulum.__use_implicit_proof = False if not os.path.isdir(Reticulum.storagepath): os.makedirs(Reticulum.storagepath) @@ -50,6 +51,8 @@ class Reticulum: RNS.Identity.loadKnownDestinations() Reticulum.router = self + RNS.Transport.scheduleJobs() + atexit.register(RNS.Identity.exitHandler) def applyConfig(self): @@ -66,6 +69,11 @@ class Reticulum: if "reticulum" in self.config: for option in self.config["reticulum"]: value = self.config["reticulum"][option] + if option == "use_implicit_proof": + if value == "true": + Reticulum.__use_implicit_proof = True + if value == "false": + Reticulum.__use_implicit_proof = False if option == "allow_unencrypted": if value == "true": RNS.log("", RNS.LOG_CRITICAL) @@ -259,4 +267,8 @@ class Reticulum: @staticmethod def should_allow_unencrypted(): - return Reticulum.__allow_unencrypted \ No newline at end of file + return Reticulum.__allow_unencrypted + + @staticmethod + def should_use_implicit_proof(): + return Reticulum.__use_implicit_proof \ No newline at end of file diff --git a/RNS/Transport.py b/RNS/Transport.py index e56fd84..ba824ba 100755 --- a/RNS/Transport.py +++ b/RNS/Transport.py @@ -1,4 +1,7 @@ import RNS +import time +import threading +from time import sleep class Transport: # Constants @@ -8,15 +11,65 @@ class Transport: TUNNEL = 0x03; types = [BROADCAST, TRANSPORT, RELAY, TUNNEL] - interfaces = [] - destinations = [] - pending_links = [] - active_links = [] - packet_hashlist = [] + interfaces = [] # All active interfaces + destinations = [] # All active destinations + pending_links = [] # Links that are being established + active_links = [] # Links that are active + packet_hashlist = [] # A list of packet hashes for duplicate detection + receipts = [] # Receipts of all outgoing packets for proof processing + + jobs_locked = False + jobs_running = False + job_interval = 0.250 + receipts_last_checked = 0.0 + receipts_check_interval = 1.0 + hashlist_maxsize = 1000000 + + @staticmethod + def scheduleJobs(): + thread = threading.Thread(target=Transport.jobloop) + thread.setDaemon(True) + thread.start() + + @staticmethod + def jobloop(): + while (True): + Transport.jobs() + sleep(Transport.job_interval) + + @staticmethod + def jobs(): + Transport.jobs_running = True + try: + if not Transport.jobs_locked: + # Process receipts list for timed-out packets + if Transport.receipts_last_checked+Transport.receipts_check_interval < time.time(): + for receipt in Transport.receipts: + receipt.checkTimeout() + if receipt.status != RNS.PacketReceipt.SENT: + Transport.receipts.remove(receipt) + + Transport.receipts_last_checked = time.time() + + # Cull the packet hashlist if it has reached max size + while (len(Transport.packet_hashlist) > Transport.hashlist_maxsize): + Transport.packet_hashlist.pop(0) + + except Exception as e: + RNS.log("An exception occurred while running Transport jobs.", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) + + Transport.jobs_running = False @staticmethod def outbound(packet): - Transport.cacheRaw(packet.raw) + while (Transport.jobs_running): + sleep(0.1) + + Transport.jobs_locked = True + packet.updateHash() + sent = False + for interface in Transport.interfaces: if interface.OUT: should_transmit = True @@ -27,19 +80,38 @@ class Transport: if should_transmit: RNS.log("Transmitting "+str(len(packet.raw))+" bytes via: "+str(interface), RNS.LOG_DEBUG) interface.processOutgoing(packet.raw) + sent = True + + if sent: + packet.sent = True + packet.sent_at = time.time() + + if (packet.packet_type == RNS.Packet.DATA): + packet.receipt = RNS.PacketReceipt(packet) + Transport.receipts.append(packet.receipt) + + Transport.cache(packet) + + Transport.jobs_locked = False + return sent @staticmethod def inbound(raw, interface=None): - packet_hash = RNS.Identity.fullHash(raw) - RNS.log(str(interface)+" received packet with hash "+RNS.prettyhexrep(packet_hash), RNS.LOG_DEBUG) + while (Transport.jobs_running): + sleep(0.1) + + Transport.jobs_locked = True + + packet = RNS.Packet(None, raw) + packet.unpack() + packet.updateHash() + packet.receiving_interface = interface - if not packet_hash in Transport.packet_hashlist: - Transport.packet_hashlist.append(packet_hash) - packet = RNS.Packet(None, raw) - packet.unpack() - packet.packet_hash = packet_hash - packet.receiving_interface = interface + RNS.log(str(interface)+" received packet with hash "+RNS.prettyhexrep(packet.packet_hash), RNS.LOG_DEBUG) + if not packet.packet_hash in Transport.packet_hashlist: + Transport.packet_hashlist.append(packet.packet_hash) + if packet.packet_type == RNS.Packet.ANNOUNCE: if RNS.Identity.validateAnnounce(packet): Transport.cache(packet) @@ -64,6 +136,13 @@ class Transport: destination.receive(packet) Transport.cache(packet) + if destination.proof_strategy == RNS.Destination.PROVE_ALL: + packet.prove() + + if destination.proof_strategy == RNS.Destination.PROVE_APP: + if destination.callbacks.proof_requested: + destination.callbacks.proof_requested(packet) + if packet.packet_type == RNS.Packet.PROOF: if packet.header_type == RNS.Packet.HEADER_3: # This is a link request proof, forward @@ -72,11 +151,27 @@ class Transport: if link.link_id == packet.destination_hash: link.validateProof(packet) else: - for destination in Transport.destinations: - if destination.hash == packet.destination_hash: - if destination.proofcallback != None: - destination.proofcallback(packet) - # TODO: add universal proof handling + # TODO: Make sure everything uses new proof handling + if len(packet.data) == RNS.PacketReceipt.EXPL_LENGTH: + proof_hash = packet.data[:RNS.Identity.HASHLENGTH/8] + else: + proof_hash = None + + for receipt in Transport.receipts: + receipt_validated = False + if proof_hash != None: + # Only test validation if hash matches + if receipt.hash == proof_hash: + receipt_validated = receipt.validateProofPacket(packet) + else: + # In case of an implicit proof, we have + # to check every single outstanding receipt + receipt_validated = receipt.validateProofPacket(packet) + + if receipt_validated: + Transport.receipts.remove(receipt) + + Transport.jobs_locked = False @staticmethod def registerDestination(destination): @@ -112,15 +207,13 @@ class Transport: @staticmethod def cache(packet): if RNS.Transport.shouldCache(packet): - RNS.Transport.cacheRaw(packet.raw) + try: + packet_hash = RNS.hexrep(packet.getHash(), delimit=False) + file = open(RNS.Reticulum.cachepath+"/"+packet_hash, "w") + file.write(packet.raw) + file.close() + RNS.log("Wrote packet "+packet_hash+" to cache", RNS.LOG_DEBUG) + except Exception as e: + RNS.log("Error writing packet to cache", RNS.LOG_ERROR) + RNS.log("The contained exception was: "+str(e)) - @staticmethod - def cacheRaw(raw): - try: - file = open(RNS.Reticulum.cachepath+"/"+RNS.hexrep(RNS.Identity.fullHash(raw), delimit=False), "w") - file.write(raw) - file.close() - RNS.log("Wrote packet "+RNS.prettyhexrep(RNS.Identity.fullHash(raw))+" to cache", RNS.LOG_DEBUG) - except Exception as e: - RNS.log("Error writing packet to cache", RNS.LOG_ERROR) - RNS.log("The contained exception was: "+str(e)) \ No newline at end of file diff --git a/RNS/__init__.py b/RNS/__init__.py index 45b4d0c..e632b60 100755 --- a/RNS/__init__.py +++ b/RNS/__init__.py @@ -9,6 +9,7 @@ from .Link import Link from .Transport import Transport from .Destination import Destination from .Packet import Packet +from .Packet import PacketReceipt modules = glob.glob(os.path.dirname(__file__)+"/*.py") __all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')] diff --git a/RNS/vendor/configobj.py b/RNS/vendor/configobj.py index 0587cd4..f6bc5a1 100755 --- a/RNS/vendor/configobj.py +++ b/RNS/vendor/configobj.py @@ -38,7 +38,7 @@ BOMS = { BOM_UTF16: ('utf_16', 'utf_16'), } # All legal variants of the BOM codecs. -# TODO: the list of aliases is not meant to be exhaustive, is there a +# The list of aliases is not meant to be exhaustive, is there a # better way ? BOM_LIST = { 'utf_16': 'utf_16',