diff --git a/horusdemodlib/__init__.py b/horusdemodlib/__init__.py index f3b4574..970659c 100755 --- a/horusdemodlib/__init__.py +++ b/horusdemodlib/__init__.py @@ -1 +1 @@ -__version__ = "0.1.15" +__version__ = "0.1.16" diff --git a/horusdemodlib/demodstats.py b/horusdemodlib/demodstats.py new file mode 100644 index 0000000..531db65 --- /dev/null +++ b/horusdemodlib/demodstats.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python +# +# Horus Binary - fsk_demod modem statistics parser +# +# Copyright (C) 2019 Mark Jessop +# Released under GNU GPL v3 or later +# +# This utility ingests fsk_demod stats output via stdin, and optionally emits time-averaged modem statistics +# data via UDP. +# +import argparse +import json +import logging +import socket +import sys +import time +import numpy as np + + +class FSKDemodStats(object): + """ + Process modem statistics produced by horus/fsk_demod and provide access to + filtered or instantaneous modem data. + + This class expects the JSON output from horus_demod to be arriving in *realtime*. + The test script below will emulate relatime input based on a file. + """ + + FSK_STATS_FIELDS = ['EbNodB', 'ppm', 'f1_est', 'f2_est', 'samp_fft'] + + + def __init__(self, + averaging_time = 5.0, + peak_hold = False, + decoder_id = "" + ): + """ + + Required Fields: + averaging_time (float): Use the last X seconds of data in calculations. + peak_hold (bool): If true, use a peak-hold SNR metric instead of a mean. + decoder_id (str): A unique ID for this object (suggest use of the SDR device ID) + + """ + + self.averaging_time = float(averaging_time) + self.peak_hold = peak_hold + self.decoder_id = str(decoder_id) + + # Input data stores. + self.in_times = np.array([]) + self.in_snr = np.array([]) + self.in_ppm = np.array([]) + + + # Output State variables. + self.snr = -999.0 + self.fest = [0.0,0.0, 0.0,0.0] + self.fft = [] + self.ppm = 0.0 + + + + def update(self, data): + """ + Update the statistics parser with a new set of output from fsk_demod. + This can accept either a string (which will be parsed as JSON), or a dict. + + Required Fields: + data (str, dict): One set of statistics from fsk_demod. + """ + + # Check input type + if type(data) == str: + # Attempt to parse string. + try: + # Clean up any nan entries, which aren't valid JSON. + # For now we just replace these with 0, since they only seem to occur + # in the eye diagram data, which we don't use anyway. + if 'nan' in data: + data = data.replace('nan', '0.0') + + _data = json.loads(data) + except Exception as e: + self.log_error("FSK Demod Stats - %s" % str(e)) + return + elif type(data) == dict: + _data = data + + else: + return + + # Check for required fields in incoming dictionary. + for _field in self.FSK_STATS_FIELDS: + if _field not in _data: + self.log_error("Missing Field %s" % _field) + return + + # Now we can process the data. + _time = time.time() + self.fft = _data['samp_fft'] + self.fest = [0.0,0.0,0.0,0.0] + self.fest[0] = _data['f1_est'] + self.fest[1] = _data['f2_est'] + + if 'f3_est' in _data: + self.fest[2] = _data['f3_est'] + + if 'f4_est' in _data: + self.fest[3] = _data['f4_est'] + else: + self.fest = self.fest[:2] + + # Time-series data + self.in_times = np.append(self.in_times, _time) + self.in_snr = np.append(self.in_snr, _data['EbNodB']) + self.in_ppm = np.append(self.in_ppm, _data['ppm']) + + + # Calculate SNR / PPM + _time_range = self.in_times>(_time-self.averaging_time) + # Clip arrays to just the values we want + self.in_ppm = self.in_ppm[_time_range] + self.in_snr = self.in_snr[_time_range] + self.in_times = self.in_times[_time_range] + + # Always just take a mean of the PPM values. + self.ppm = np.mean(self.in_ppm) + + if self.peak_hold: + self.snr = np.max(self.in_snr) + else: + self.snr = np.mean(self.in_snr) + + + def log_debug(self, line): + """ Helper function to log a debug message with a descriptive heading. + Args: + line (str): Message to be logged. + """ + logging.debug("FSK Demod Stats #%s - %s" % (str(self.decoder_id), line)) + + + def log_info(self, line): + """ Helper function to log an informational message with a descriptive heading. + Args: + line (str): Message to be logged. + """ + logging.info("FSK Demod Stats #%s - %s" % (str(self.decoder_id), line)) + + + def log_error(self, line): + """ Helper function to log an error message with a descriptive heading. + Args: + line (str): Message to be logged. + """ + logging.error("FSK Demod Stats #%s - %s" % (str(self.decoder_id), line)) + + + +def send_modem_stats(stats, udp_port=55672): + """ Send a JSON-encoded dictionary to the wenet frontend """ + try: + s = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) + s.settimeout(1) + # Set up socket for broadcast, and allow re-use of the address + s.setsockopt(socket.SOL_SOCKET,socket.SO_BROADCAST,1) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + except: + pass + s.bind(('',udp_port)) + try: + s.sendto(json.dumps(stats).encode('ascii'), ('', udp_port)) + except socket.error: + s.sendto(json.dumps(stats).encode('ascii'), ('127.0.0.1', udp_port)) + + except Exception as e: + logging.error("Error updating GUI with modem status: %s" % str(e)) + + + +if __name__ == "__main__": + # Command line arguments. + parser = argparse.ArgumentParser() + parser.add_argument("-r", "--rate", default=1, type=int, help="Update Rate (Hz). Default: 2 Hz") + parser.add_argument("-p", "--port", default=55672, type=int, help="Output UDP port. Default: 55672") + parser.add_argument("-s", "--source", default='MFSK', help="Source name (must be unique if running multiple decoders). Default: MFSK") + args = parser.parse_args() + + _averaging_time = 1.0/args.rate + + stats_parser = FSKDemodStats(averaging_time=_averaging_time, peak_hold=True) + + + _last_update_time = time.time() + + try: + while True: + data = sys.stdin.readline() + + # An empty line indicates that stdin has been closed. + if data == '': + break + + # Otherwise, feed it to the stats parser. + stats_parser.update(data.rstrip()) + + if (time.time() - _last_update_time) > _averaging_time: + # Send latest modem stats to the Wenet frontend. + _stats = { + 'type': 'MODEM_STATS', + 'source': args.source, + 'snr': stats_parser.snr, + 'ppm': stats_parser.ppm, + 'fft': stats_parser.fft, + 'fest': stats_parser.fest + } + + send_modem_stats(_stats, args.port) + + _last_update_time = time.time() + + except KeyboardInterrupt: + pass + diff --git a/horusdemodlib/habitat.py b/horusdemodlib/habitat.py new file mode 100644 index 0000000..d5e3a3d --- /dev/null +++ b/horusdemodlib/habitat.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python +# +# Horus Demod Library - Habitat Uploader +# +# Mark Jessop +# + +import datetime +import json +import logging +import random +import requests +import time +from base64 import b64encode +from hashlib import sha256 +from queue import Queue +from threading import Thread + + +class HabitatUploader(object): + """ + Queued Habitat Telemetry Uploader class + + Packets to be uploaded to Habitat are added to a queue for uploading. + If an upload attempt times out, the packet is discarded. + If the queue fills up (probably indicating no network connection, and a fast packet downlink rate), + it is immediately emptied, to avoid upload of out-of-date packets. + """ + + HABITAT_URL = "http://habitat.habhub.org/" + HABITAT_DB = "habitat" + HABITAT_UUIDS = HABITAT_URL + "_uuids?count=%d" + HABITAT_DB_URL = HABITAT_URL + HABITAT_DB + "/" + + def __init__( + self, + user_callsign="FSK_DEMOD", + listener_lat=0.0, + listener_lon=0.0, + listener_radio="", + listener_antenna="", + queue_size=64, + upload_timeout=10, + upload_retries=5, + upload_retry_interval=0.25, + inhibit=False, + ): + """ Create a Habitat Uploader object. """ + + self.upload_timeout = upload_timeout + self.upload_retries = upload_retries + self.upload_retry_interval = upload_retry_interval + self.queue_size = queue_size + self.habitat_upload_queue = Queue(queue_size) + self.inhibit = inhibit + + # Listener information + self.user_callsign = user_callsign + self.listener_lat = listener_lat + self.listener_lon = listener_lon + self.listener_radio = listener_radio + self.listener_antenna = listener_antenna + self.position_uploaded = False + + self.callsign_init = False + self.uuids = [] + + if self.inhibit: + logging.info("Habitat Uploader inhibited.") + + # Start the uploader thread. + self.habitat_uploader_running = True + self.uploadthread = Thread(target=self.habitat_upload_thread) + self.uploadthread.start() + + def habitat_upload(self, sentence): + """ Upload a UKHAS-standard telemetry sentence to Habitat """ + + # Generate payload to be uploaded + # b64encode accepts and returns bytes objects. + _sentence_b64 = b64encode(sentence.encode("ascii")) + _date = datetime.datetime.utcnow().isoformat("T") + "Z" + _user_call = self.user_callsign + + _data = { + "type": "payload_telemetry", + "data": { + "_raw": _sentence_b64.decode( + "ascii" + ) # Convert back to a string to be serialisable + }, + "receivers": { + _user_call: {"time_created": _date, "time_uploaded": _date,}, + }, + } + + # The URl to upload to. + _url = f"{self.HABITAT_URL}{self.HABITAT_DB}/_design/payload_telemetry/_update/add_listener/{sha256(_sentence_b64).hexdigest()}" + + # Delay for a random amount of time between 0 and upload_retry_interval*2 seconds. + time.sleep(random.random() * self.upload_retry_interval * 2.0) + + _retries = 0 + + # When uploading, we have three possible outcomes: + # - Can't connect. No point re-trying in this situation. + # - The packet is uploaded successfult (201 / 403) + # - There is a upload conflict on the Habitat DB end (409). We can retry and it might work. + while _retries < self.upload_retries: + # Run the request. + try: + _req = requests.put( + _url, data=json.dumps(_data), timeout=self.upload_timeout + ) + except Exception as e: + logging.error("Habitat - Upload Failed: %s" % str(e)) + break + + if _req.status_code == 201 or _req.status_code == 403: + # 201 = Success, 403 = Success, sentence has already seen by others. + logging.info(f"Habitat - Uploaded sentence: {sentence.strip()}") + _upload_success = True + break + elif _req.status_code == 409: + # 409 = Upload conflict (server busy). Sleep for a moment, then retry. + logging.debug("Habitat - Upload conflict.. retrying.") + time.sleep(random.random() * self.upload_retry_interval) + _retries += 1 + else: + logging.error( + "Habitat - Error uploading to Habitat. Status Code: %d." + % _req.status_code + ) + break + + if _retries == self.upload_retries: + logging.error( + "Habitat - Upload conflict not resolved with %d retries." + % self.upload_retries + ) + + return + + def habitat_upload_thread(self): + """ Handle uploading of packets to Habitat """ + + logging.info("Started Habitat Uploader Thread.") + + while self.habitat_uploader_running: + + if self.habitat_upload_queue.qsize() > 0: + # If the queue is completely full, jump to the most recent telemetry sentence. + if self.habitat_upload_queue.qsize() == self.queue_size: + while not self.habitat_upload_queue.empty(): + sentence = self.habitat_upload_queue.get() + + logging.warning( + "Habitat uploader queue was full - possible connectivity issue." + ) + else: + # Otherwise, get the first item in the queue. + sentence = self.habitat_upload_queue.get() + + # Attempt to upload it. + self.habitat_upload(sentence) + + else: + # Wait for a short time before checking the queue again. + time.sleep(0.5) + + if not self.position_uploaded: + # Validate the lat/lon entries. + try: + _lat = float(self.listener_lat) + _lon = float(self.listener_lon) + + if (_lat != 0.0) or (_lon != 0.0): + _success = self.uploadListenerPosition( + self.user_callsign, + _lat, + _lon, + self.listener_radio, + self.listener_antenna, + ) + else: + logging.warning("Listener position set to 0.0/0.0 - not uploading.") + + except Exception as e: + logging.error("Error uploading listener position: %s" % str(e)) + + # Set this flag regardless if the upload worked. + # The user can trigger a re-upload. + self.position_uploaded = True + + + logging.info("Stopped Habitat Uploader Thread.") + + def add(self, sentence): + """ Add a sentence to the upload queue """ + + if self.inhibit: + # We have upload inhibited. Return. + return + + # Handling of arbitrary numbers of $$'s at the start of a sentence: + # Extract the data part of the sentence (i.e. everything after the $$'s') + sentence = sentence.split("$")[-1] + # Now add the *correct* number of $$s back on. + sentence = "$$" + sentence + + if not (sentence[-1] == "\n"): + sentence += "\n" + + try: + self.habitat_upload_queue.put_nowait(sentence) + except Exception as e: + logging.error("Error adding sentence to queue: %s" % str(e)) + + def close(self): + """ Shutdown uploader thread. """ + self.habitat_uploader_running = False + + def ISOStringNow(self): + return "%sZ" % datetime.datetime.utcnow().isoformat() + + def postListenerData(self, doc, timeout=10): + + # do we have at least one uuid, if not go get more + if len(self.uuids) < 1: + self.fetchUuids() + + # Attempt to add UUID and time data to document. + try: + doc["_id"] = self.uuids.pop() + except IndexError: + logging.error( + "Habitat - Unable to post listener data - no UUIDs available." + ) + return False + + doc["time_uploaded"] = self.ISOStringNow() + + try: + _r = requests.post( + f"{self.HABITAT_URL}{self.HABITAT_DB}/", json=doc, timeout=timeout + ) + return True + except Exception as e: + logging.error("Habitat - Could not post listener data - %s" % str(e)) + return False + + def fetchUuids(self, timeout=10): + + _retries = 5 + + while _retries > 0: + try: + _r = requests.get(self.HABITAT_UUIDS % 10, timeout=timeout) + self.uuids.extend(_r.json()["uuids"]) + logging.debug("Habitat - Got UUIDs") + return + except Exception as e: + logging.error( + "Habitat - Unable to fetch UUIDs, retrying in 2 seconds - %s" + % str(e) + ) + time.sleep(2) + _retries = _retries - 1 + continue + + logging.error("Habitat - Gave up trying to get UUIDs.") + return + + def initListenerCallsign(self, callsign, radio="", antenna=""): + doc = { + "type": "listener_information", + "time_created": self.ISOStringNow(), + "data": {"callsign": callsign, "antenna": antenna, "radio": radio,}, + } + + resp = self.postListenerData(doc) + + if resp is True: + logging.debug("Habitat - Listener Callsign Initialized.") + return True + else: + logging.error("Habitat - Unable to initialize callsign.") + return False + + def uploadListenerPosition(self, callsign, lat, lon, radio="", antenna=""): + """ Initializer Listener Callsign, and upload Listener Position """ + + # Attempt to initialize the listeners callsign + resp = self.initListenerCallsign(callsign, radio=radio, antenna=antenna) + # If this fails, it means we can't contact the Habitat server, + # so there is no point continuing. + if resp is False: + return False + + doc = { + "type": "listener_telemetry", + "time_created": self.ISOStringNow(), + "data": { + "callsign": callsign, + "chase": False, + "latitude": lat, + "longitude": lon, + "altitude": 0, + "speed": 0, + }, + } + + # post position to habitat + resp = self.postListenerData(doc) + if resp is True: + logging.info("Habitat - Listener information uploaded.") + return True + else: + logging.error("Habitat - Unable to upload listener information.") + return False + + def trigger_position_upload(self): + """ Trigger a re-upload of the listener position """ + self.position_uploaded = False + + +if __name__ == "__main__": + + # Setup Logging + logging.basicConfig( + format="%(asctime)s %(levelname)s: %(message)s", level=logging.INFO + ) + + habitat = HabitatUploader( + user_callsign="HORUSGUI_TEST", + listener_lat=-34.0, + listener_lon=138.0, + listener_radio="Testing Habitat Uploader", + listener_antenna="Wet Noodle", + ) + + habitat.add("$$DUMMY,0,0.0,0.0*F000") + + time.sleep(10) + habitat.trigger_position_upload() + time.sleep(5) + habitat.close() diff --git a/horusdemodlib/payloads.py b/horusdemodlib/payloads.py index 307d0e6..55e70a3 100644 --- a/horusdemodlib/payloads.py +++ b/horusdemodlib/payloads.py @@ -142,6 +142,8 @@ def init_payload_id_list(filename="payload_id_list.txt"): else: logging.warning("Could not download Payload ID List - attempting to use local version.") HORUS_PAYLOAD_LIST = read_payload_list(filename=filename) + + return HORUS_PAYLOAD_LIST @@ -272,6 +274,8 @@ def init_custom_field_list(filename="custom_field_list.json"): else: logging.warning("Could not download Custom Field List - attempting to use local version.") HORUS_CUSTOM_FIELDS = read_custom_field_list(filename=filename) + + return HORUS_CUSTOM_FIELDS def update_payload_lists(payload_list, custom_field_list): diff --git a/horusdemodlib/uploader.py b/horusdemodlib/uploader.py new file mode 100644 index 0000000..b273fa5 --- /dev/null +++ b/horusdemodlib/uploader.py @@ -0,0 +1,193 @@ +# +# HorusLib - Command-Line Uploader +# + +# Python 3 check +import sys + +if sys.version_info < (3, 6): + print("ERROR - This script requires Python 3.6 or newer!") + sys.exit(1) + +import argparse +import codecs +import traceback +from configparser import RawConfigParser + +from .habitat import * +from .decoder import decode_packet, parse_ukhas_string +from .payloads import * +from .horusudp import send_payload_summary +from .payloads import init_custom_field_list, init_payload_id_list +from .demodstats import FSKDemodStats +import horusdemodlib.payloads + +def read_config(filename): + ''' Read in the user configuation file.''' + user_config = { + 'user_call' : 'HORUS_RX', + 'ozi_udp_port' : 55683, + 'summary_port' : 55672, + 'station_lat' : 0.0, + 'station_lon' : 0.0, + 'radio_comment' : "", + 'antenna_comment' : "" + } + + try: + config = RawConfigParser() + config.read(filename) + + user_config['user_call'] = config.get('user', 'callsign') + user_config['station_lat'] = config.getfloat('user', 'station_lat') + user_config['station_lon'] = config.getfloat('user', 'station_lon') + user_config['radio_comment'] = config.get('user', 'radio_comment') + user_config['antenna_comment'] = config.get('user', 'antenna_comment') + user_config['ozi_udp_port'] = config.getint('horus_udp', 'ozimux_port') + user_config['summary_port'] = config.getint('horus_udp', 'summary_port') + + return user_config + + except: + traceback.print_exc() + logging.error("Could not parse config file, exiting. Have you copied user.cfg.example to user.cfg?") + return None + + +def main(): + + # Read command-line arguments + parser = argparse.ArgumentParser(description="Project Horus Binary/RTTY Telemetry Handler", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('-c', '--config', type=str, default='user.cfg', help="Configuration file to use. Default: user.cfg") + parser.add_argument("--noupload", action="store_true", default=False, help="Disable Habitat upload.") + parser.add_argument("--rtty", action="store_true", default=False, help="Expect only RTTY inputs, do not update payload lists.") + parser.add_argument("--log", type=str, default="telemetry.log", help="Write decoded telemetry to this log file.") + parser.add_argument("--debuglog", type=str, default="horusb_debug.log", help="Write debug log to this file.") + parser.add_argument("--payload-list", type=str, default="payload_id_list.txt", help="List of known payload IDs.") + parser.add_argument("--custom-fields", type=str, default="custom_field_list.json", help="List of payload Custom Fields") +# parser.add_argument("--ozimux", type=int, default=-1, help="Override user.cfg OziMux output UDP port. (NOT IMPLEMENTED)") +# parser.add_argument("--summary", type=int, default=-1, help="Override user.cfg UDP Summary output port. (NOT IMPLEMENTED)") + parser.add_argument("-v", "--verbose", action="store_true", default=False, help="Verbose output (set logging level to DEBUG)") + args = parser.parse_args() + + if args.verbose: + logging_level = logging.DEBUG + else: + logging_level = logging.INFO + + # Set up logging + logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging_level) + + # Read in the configuration file. + user_config = read_config(args.config) + + # If we could not read the configuration file, exit. + if user_config == None: + logging.critical(f"Could not load {args.config}, exiting...") + sys.exit(1) + + + if args.rtty == False: + # Initialize Payload List + horusdemodlib.payloads.HORUS_PAYLOAD_LIST = init_payload_id_list(filename=args.payload_list) + + logging.info(f"Payload list contains {len(list(horusdemodlib.payloads.HORUS_PAYLOAD_LIST.keys()))} entries.") + + # Init Custom Fields List + horusdemodlib.payloads.HORUS_CUSTOM_FIELDS = init_custom_field_list(filename=args.custom_fields) + logging.info(f"Custom Field list contains {len(list(horusdemodlib.payloads.HORUS_CUSTOM_FIELDS.keys()))} entries.") + + # Start the Habitat uploader thread. + habitat_uploader = HabitatUploader( + user_callsign = user_config['user_call'], + listener_lat = user_config['station_lat'], + listener_lon = user_config['station_lon'], + listener_radio = user_config['radio_comment'], + listener_antenna = user_config['antenna_comment'], + inhibit=args.noupload + ) + + logging.info("Using User Callsign: %s" % user_config['user_call']) + + demod_stats = FSKDemodStats() + + logging.info("Started Horus Demod Uploader. Hit CTRL-C to exit.") + # Main loop + try: + while True: + # Read lines in from stdin, and strip off any trailing newlines + data = sys.stdin.readline() + + if (data == ''): + # Empty line means stdin has been closed. + logging.info("Caught EOF, exiting.") + break + + # Otherwise, strip any newlines, and continue. + data = data.rstrip() + + # If the line of data starts with '$$', we assume it is a UKHAS-standard ASCII telemetry sentence. + # Otherwise, we assume it is a string of hexadecimal bytes, and attempt to parse it as a binary telemetry packet. + + if data.startswith('$$'): + # RTTY packet handling. + # Attempt to extract fields from it: + logging.info(f"Received raw RTTY packet: {data}") + try: + _decoded = parse_ukhas_string(data) + # If we get here, the string is valid! + + # Add in SNR data. + _snr = demod_stats.snr + _decoded['snr'] = _snr + + # Send via UDP + send_payload_summary(_decoded, port=user_config['summary_port']) + + # Upload the string to Habitat + _decoded_str = "$$" + data.split('$')[-1] + '\n' + habitat_uploader.add(_decoded_str) + + logging.info(f"Decoded String (SNR {demod_stats.snr:.1f} dB): {_decoded_str[:-1]}") + + except Exception as e: + logging.error(f"Decode Failed: {str(e)}") + + elif data.startswith('{'): + # Possibly a line of modem statistics, attempt to decode it. + demod_stats.update(data) + + else: + # Handle binary packets + logging.info(f"Received raw binary packet: {data}") + try: + _binary_string = codecs.decode(data, 'hex') + except TypeError as e: + logging.error("Error parsing line as hexadecimal (%s): %s" % (str(e), data)) + continue + + try: + _decoded = decode_packet(_binary_string) + # If we get here, we have a valid packet! + + # Add in SNR data. + _snr = demod_stats.snr + _decoded['snr'] = _snr + + # Send via UDP + send_payload_summary(_decoded, port=user_config['summary_port']) + + # Upload to Habitat + habitat_uploader.add(_decoded['ukhas_str']+'\n') + + logging.info(f"Decoded Binary Packet (SNR {demod_stats.snr:.1f} dB): {_decoded['ukhas_str']}") + except Exception as e: + logging.error(f"Decode Failed: {str(e)}") + + except KeyboardInterrupt: + logging.info("Caught CTRL-C, exiting.") + + habitat_uploader.close() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 14868ad..d7cfe18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "horusdemodlib" -version = "0.1.15" +version = "0.1.16" description = "Project Horus HAB Telemetry Demodulators" authors = ["Mark Jessop"] license = "LGPL-2.1-or-later" @@ -9,6 +9,7 @@ license = "LGPL-2.1-or-later" python = "^3.6" requests = "^2.24.0" crcmod = "^1.7" +numpy = "^1.17" [tool.poetry.dev-dependencies] diff --git a/requirements.txt b/requirements.txt index 4e67c28..c80b783 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests -crcmod \ No newline at end of file +crcmod +numpy \ No newline at end of file diff --git a/src/horus_demod.c b/src/horus_demod.c index daf5461..22b5900 100644 --- a/src/horus_demod.c +++ b/src/horus_demod.c @@ -47,9 +47,9 @@ int main(int argc, char *argv[]) { struct horus *hstates; struct MODEM_STATS stats; - FILE *fin,*fout; + FILE *fin,*fout,*stats_outfile; int i,j,Ndft,mode; - int stats_ctr,stats_loop, stats_rate, verbose, crc_results; + int stats_ctr,stats_loop, stats_rate, verbose, crc_results, stdout_stats; float loop_time; int enable_stats = 0; int quadrature = 0; @@ -61,7 +61,7 @@ int main(int argc, char *argv[]) { stats_loop = 0; stats_rate = 8; mode = -1; - verbose = crc_results = 0; + verbose = crc_results = stdout_stats = 0; int o = 0; int opt_idx = 0; @@ -77,7 +77,7 @@ int main(int argc, char *argv[]) { {0, 0, 0, 0} }; - o = getopt_long(argc,argv,"hvcqm:t::",long_opts,&opt_idx); + o = getopt_long(argc,argv,"hvcgqm:t::",long_opts,&opt_idx); switch(o) { case 'm': @@ -118,7 +118,10 @@ int main(int argc, char *argv[]) { break; case 'c': crc_results = 1; - break; + break; + case 'g': + stdout_stats = 1; + break; case 'h': case '?': goto helpmsg; @@ -170,6 +173,7 @@ int main(int argc, char *argv[]) { fprintf(stderr," -t[r] --stats=[r] Print out modem statistics to stderr in JSON.\n"); fprintf(stderr," r, if provided, sets the number of modem frames\n" " between statistic printouts\n"); + fprintf(stderr," -g Emit Stats on stdout instead of stderr\n"); fprintf(stderr," -q use stereo (IQ) input\n"); fprintf(stderr," -v verbose debug info\n"); fprintf(stderr," -c display CRC results for each packet\n"); @@ -198,6 +202,12 @@ int main(int argc, char *argv[]) { exit(1); } + if (stdout_stats){ + stats_outfile = stdout; + } else { + stats_outfile = stderr; + } + /* end command line processing */ hstates = horus_open_advanced(mode, Rs, tone_spacing); @@ -252,30 +262,30 @@ int main(int argc, char *argv[]) { /* Print standard 2FSK stats */ - fprintf(stderr,"{\"EbNodB\": %2.2f,\t\"ppm\": %d,",stats.snr_est, (int)stats.clock_offset); - fprintf(stderr,"\t\"f1_est\":%.1f,\t\"f2_est\":%.1f",stats.f_est[0], stats.f_est[1]); + fprintf(stats_outfile,"{\"EbNodB\": %2.2f,\t\"ppm\": %d,",stats.snr_est, (int)stats.clock_offset); + fprintf(stats_outfile,"\t\"f1_est\":%.1f,\t\"f2_est\":%.1f",stats.f_est[0], stats.f_est[1]); /* Print 4FSK stats if in 4FSK mode */ if (horus_get_mFSK(hstates) == 4) { - fprintf(stderr,",\t\"f3_est\":%.1f,\t\"f4_est\":%.1f", stats.f_est[2], stats.f_est[3]); + fprintf(stats_outfile,",\t\"f3_est\":%.1f,\t\"f4_est\":%.1f", stats.f_est[2], stats.f_est[3]); } /* Print the eye diagram */ - fprintf(stderr,",\t\"eye_diagram\":["); + fprintf(stats_outfile,",\t\"eye_diagram\":["); for(i=0;i /dev/null; then + echo "Found bc." +else + echo "ERROR - Cannot find bc - Did you install it?" + exit 1 +fi + +# Use a local venv if it exists +VENV_DIR=venv +if [ -d "$VENV_DIR" ]; then + echo "Entering venv." + source $VENV_DIR/bin/activate +fi + + +# Calculate the frequency estimator limits +# Note - these are somewhat hard-coded for this dual-RX application. +RTTY_LOWER=$(echo "$RTTY_SIGNAL - $RXBANDWIDTH/2" | bc) +RTTY_UPPER=$(echo "$RTTY_SIGNAL + $RXBANDWIDTH/2" | bc) + +MFSK_LOWER=$(echo "$MFSK_SIGNAL - $RXBANDWIDTH/2" | bc) +MFSK_UPPER=$(echo "$MFSK_SIGNAL + $RXBANDWIDTH/2" | bc) + +echo "Using SDR Centre Frequency: $RXFREQ Hz." +echo "Using RTTY estimation range: $RTTY_LOWER - $RTTY_UPPER Hz" +echo "Using MFSK estimation range: $MFSK_LOWER - $MFSK_UPPER Hz" + +BIAS_SETTING="" + +if [ "$BIAS" = "1" ]; then + echo "Enabling Bias Tee." + BIAS_SETTING=" -T" +fi + +GAIN_SETTING="" +if [ "$GAIN" = "0" ]; then + echo "Using AGC." + GAIN_SETTING="" +else + echo "Using Manual Gain" + GAIN_SETTING=" -g $GAIN" +fi + +STATS_SETTING="" + +if [ "$STATS_OUTPUT" = "1" ]; then + echo "Enabling Modem Statistics." + STATS_SETTING=" --stats=100" +fi + +# Start the receive chain. +rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $RXFREQ | tee >($DECODER -q --stats=5 -g -m RTTY --fsk_lower=$RTTY_LOWER --fsk_upper=$RTTY_UPPER - - | python -m horusdemodlib.uploader --rtty) >($DECODER -q --stats=5 -g -m binary --fsk_lower=$MFSK_LOWER --fsk_upper=$MFSK_UPPER - - | python -m horusdemodlib.uploader) > /dev/null diff --git a/start_gqrx.sh b/start_gqrx.sh new file mode 100755 index 0000000..298fcfc --- /dev/null +++ b/start_gqrx.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# Horus Binary GQRX Helper Script +# +# Accepts data from GQRX's UDP output, and passes it into horus_demod. +# + +# Decoder mode. +# Can be: 'binary', 'rtty', '256bit' or '128bit' +MODE="binary" + +# Check that the horus_demod decoder has been compiled. +DECODER=./build/src/horus_demod +if [ -f "$DECODER" ]; then + echo "Found horus_demod." +else + echo "ERROR - $DECODER does not exist - have you compiled it yet?" + exit 1 +fi + +# Use a local venv if it exists +VENV_DIR=venv +if [ -d "$VENV_DIR" ]; then + echo "Entering venv." + source $VENV_DIR/bin/activate +fi + + +if [[ $OSTYPE == darwin* ]]; then + # OSX's netcat utility uses a different, incompatible syntax. Sigh. + nc -l -u localhost 7355 | $DECODER -m $MODE --stats=5 -g --fsk_lower=100 --fsk_upper=20000 - - | python -m horusdemodlib.uploader $@ +else + # Start up! + nc -l -u -p 7355 localhost | $DECODER -m $MODE --stats=5 -g --fsk_lower=100 --fsk_upper=20000 - - | python -m horusdemodlib.uploader $@ +fi diff --git a/start_rtlsdr.sh b/start_rtlsdr.sh new file mode 100755 index 0000000..a400a72 --- /dev/null +++ b/start_rtlsdr.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# Horus Binary RTLSDR Helper Script +# +# Uses rtl_fm to receive a chunk of spectrum, and passes it into horus_demod. +# + +# Receive *centre* frequency, in Hz +# Note: The SDR will be tuned to RXBANDWIDTH/2 below this frequency. +RXFREQ=434660000 + +# Receiver Gain. Set this to 0 to use automatic gain control, otherwise if running a +# preamplifier, you may want to experiment with different gain settings to optimize +# your receiver setup. +# You can find what gain range is valid for your RTLSDR by running: rtl_test +GAIN=0 + +# Bias Tee Enable (1) or Disable (0) +BIAS=0 + +# Receiver PPM offset +PPM=0 + +# Frequency estimator bandwidth. The wider the bandwidth, the more drift and frequency error the modem can tolerate, +# but the higher the chance that the modem will lock on to a strong spurious signal. +# Note: The SDR will be tuned to RXFREQ-RXBANDWIDTH/2, and the estimator set to look at 0-RXBANDWIDTH Hz. +RXBANDWIDTH=10000 + +# Enable (1) or disable (0) modem statistics output. +# If enabled, modem statistics are written to stats.txt, and can be observed +# during decoding by running: tail -f stats.txt | python fskstats.py +STATS_OUTPUT=0 + + +# Check that the horus_demod decoder has been compiled. +DECODER=./build/src/horus_demod +if [ -f "$DECODER" ]; then + echo "Found horus_demod." +else + echo "ERROR - $DECODER does not exist - have you compiled it yet?" + exit 1 +fi + +# Check that bc is available on the system path. +if echo "1+1" | bc > /dev/null; then + echo "Found bc." +else + echo "ERROR - Cannot find bc - Did you install it?" + exit 1 +fi + +# Use a local venv if it exists +VENV_DIR=venv +if [ -d "$VENV_DIR" ]; then + echo "Entering venv." + source $VENV_DIR/bin/activate +fi + +# Calculate the SDR tuning frequency +SDR_RX_FREQ=$(echo "$RXFREQ - $RXBANDWIDTH/2 - 1000" | bc) + +# Calculate the frequency estimator limits +FSK_LOWER=1000 +FSK_UPPER=$(echo "$FSK_LOWER + $RXBANDWIDTH" | bc) + +echo "Using SDR Centre Frequency: $SDR_RX_FREQ Hz." +echo "Using FSK estimation range: $FSK_LOWER - $FSK_UPPER Hz" + +BIAS_SETTING="" + +if [ "$BIAS" = "1" ]; then + echo "Enabling Bias Tee." + BIAS_SETTING=" -T" +fi + +GAIN_SETTING="" +if [ "$GAIN" = "0" ]; then + echo "Using AGC." + GAIN_SETTING="" +else + echo "Using Manual Gain" + GAIN_SETTING=" -g $GAIN" +fi + +# Start the receive chain. +rtl_fm -M raw -F9 -s 48000 -p $PPM $GAIN_SETTING$BIAS_SETTING -f $SDR_RX_FREQ | $DECODER -q --stats=5 -g -m binary --fsk_lower=$FSK_LOWER --fsk_upper=$FSK_UPPER - - | python -m horusdemodlib.uploader $@ diff --git a/user.cfg.example b/user.cfg.example new file mode 100644 index 0000000..9232207 --- /dev/null +++ b/user.cfg.example @@ -0,0 +1,27 @@ +# +# Horus Binary Uploader Example Configuration File +# + +[user] +# Your callsign - used when uploading to the HabHub Tracker. +callsign = YOUR_CALL_HERE + +# Your station latitude/longitude, which will show up on tracker.habhub.org. +# These values must be in Decimal Degree format. +# Leave the lat/lon at 0.0 if you do not wish your station plotted on the map, +# or if you are uploading your position via other means (i.e. using chasemapper) +station_lat = 0.0 +station_lon = 0.0 +# Radio/Antenna descriptions. +# An optional short description of your radio/antenna setup. +radio_comment = HorusDemodLib + Your Radio Description Here +antenna_comment = Your Antenna Description Here + + +[horus_udp] +# Horus-UDP Message Output port. This is the preferred output for use with mapping systems +# such as ChaseMapper. +summary_port = 55672 + +# OziMux UDP Broadcast port, for use with other mapping systems. +ozimux_port = 55683 \ No newline at end of file