Initial integration of horuslib

pull/13/head
Mark Jessop 2020-06-29 21:12:41 +09:30
rodzic 4e5e2e127a
commit b6fc2f604b
5 zmienionych plików z 250 dodań i 86 usunięć

Wyświetl plik

@ -3,7 +3,7 @@
Telemetry demodulator for the following modems in use by Project Horus
* Horus Binary Modes
* v1 - Legacy 22 byte mode, Golay FEC
* v2 - 16/32-byte modes, LDPC FEC
* v2 - 16/32-byte modes, LDPC FEC (Still in development)
* RTTY (7N2 only)
@ -14,11 +14,11 @@ Written by Mark Jessop <vk5qi@rfhead.net>
### TODO LIST - Important Stuff
* Audio input via pyAudio and spectrum display. - DONE
* Integrate Horus Modems (need help from @xssfox!)
* Basic display of decoded data (RTTY or HEX data for binary)
* Integrate Horus Modems (need help from @xssfox!) - First pass DONE
* Basic display of decoded data (RTTY or HEX data for binary) - DONE
* Save/Reload settings to file - Initial pass done.
* Decode horus binary data (move horusbinary.py into a library?)
* Upload telemetry to Habitat, with upload status
* Upload telemetry to Habitat, with upload status - DONE
* Better build system (build horuslib as part of package build?)
* Windows binary
@ -29,7 +29,7 @@ Written by Mark Jessop <vk5qi@rfhead.net>
## Usage
### Dependencies
* [horuslib](https://github.com/projecthorus/horuslib) built, and libhorus.so available either on the system path, or in this directory.
* [horuslib](https://github.com/projecthorus/horuslib) built, and libhorus.so available in this directory.
### Create a Virtual Environment

Wyświetl plik

@ -67,7 +67,7 @@ def populate_sample_rates(widgets):
class AudioStream(object):
""" Start up a pyAudio input stream, and pass data around to different callbacks """
def __init__(self, audio_device, fs, block_size=8192, fft_input=None, modem=None):
def __init__(self, audio_device, fs, block_size=8192, fft_input=None, modem=None, stats_callback = None):
self.audio_device = audio_device
self.fs = fs
@ -76,6 +76,7 @@ class AudioStream(object):
self.fft_input = fft_input
self.modem = modem
self.stats_callback = stats_callback
# Start audio stream
self.audio = pyaudio.PyAudio()
@ -97,7 +98,13 @@ class AudioStream(object):
if self.fft_input:
self.fft_input(data)
# TODO: Handle modem sample input.
if self.modem:
# Add samples to modem
_stats = self.modem.add_samples(data)
# Send any stats data back to the stats callback
if _stats:
if self.stats_callback:
self.stats_callback(_stats)
return (None, pyaudio.paContinue)

Wyświetl plik

@ -29,6 +29,7 @@ from .fft import *
from .modem import *
from .config import *
from .habitat import *
from .horuslib import HorusLib, Mode
from . import __version__
# Setup Logging
@ -72,7 +73,8 @@ d0_modem = Dock("Modem", size=(300, 80))
d0_habitat = Dock("Habitat", size=(300, 200))
d0_other = Dock("Other", size=(300, 100))
d1 = Dock("Spectrum", size=(800, 400))
d2 = Dock("Modem Stats", size=(800, 300))
d2_stats = Dock("Modem Stats", size=(70, 300))
d2_snr = Dock("SNR", size=(730, 300))
d3 = Dock("Data", size=(800, 50))
d4 = Dock("Log", size=(800, 150))
# Arrange docks.
@ -81,9 +83,10 @@ area.addDock(d1, "right", d0)
area.addDock(d0_modem, "bottom", d0)
area.addDock(d0_habitat, "bottom", d0_modem)
area.addDock(d0_other, "below", d0_habitat)
area.addDock(d2, "bottom", d1)
area.addDock(d3, "bottom", d2)
area.addDock(d2_stats, "bottom", d1)
area.addDock(d3, "bottom", d2_stats)
area.addDock(d4, "bottom", d3)
area.addDock(d2_snr, "right", d2_stats)
d0_habitat.raiseDock()
@ -196,21 +199,25 @@ widgets["estimatorLines"] = [
pos=-1000,
pen=pg.mkPen(color="w", width=2, style=QtCore.Qt.DashLine),
label="F1",
labelOpts={'position':0.9}
),
pg.InfiniteLine(
pos=-1000,
pen=pg.mkPen(color="w", width=2, style=QtCore.Qt.DashLine),
label="F2",
labelOpts={'position':0.9}
),
pg.InfiniteLine(
pos=-1000,
pen=pg.mkPen(color="w", width=2, style=QtCore.Qt.DashLine),
label="F3",
labelOpts={'position':0.9}
),
pg.InfiniteLine(
pos=-1000,
pen=pg.mkPen(color="w", width=2, style=QtCore.Qt.DashLine),
label="F4",
labelOpts={'position':0.9}
),
]
for _line in widgets["estimatorLines"]:
@ -221,28 +228,44 @@ widgets["spectrumPlot"].setLabel("bottom", "Frequency", units="Hz")
widgets["spectrumPlot"].setXRange(100, 4000)
widgets["spectrumPlot"].setYRange(-100, -20)
widgets["spectrumPlot"].setLimits(xMin=100, xMax=4000, yMin=-120, yMax=0)
widgets["spectrumPlot"].showGrid(True, True)
d1.addWidget(widgets["spectrumPlot"])
widgets["spectrumPlotRange"] = [-100, -20]
# Waterfall - TBD
w3 = pg.LayoutWidget()
w3_stats = pg.LayoutWidget()
widgets["snrLabel"] = QtGui.QLabel("<b>SNR:</b> --.- dB")
widgets["snrLabel"].setWordWrap(True);
widgets["snrLabel"].setFont(QtGui.QFont("Courier New", 18))
w3_stats.addWidget(widgets["snrLabel"], 0, 0, 2, 1)
d2_stats.addWidget(w3_stats)
# SNR Plot
w3_snr = pg.LayoutWidget()
widgets["snrPlot"] = pg.PlotWidget(title="SNR")
widgets["snrPlot"].setLabel("left", "SNR (dB)")
widgets["snrPlot"].setLabel("bottom", "Time (s)")
widgets["snrPlot"].setXRange(-60, 0)
widgets["snrPlot"].setYRange(-10, 30)
widgets["snrPlot"].setLimits(xMin=0, xMax=60, yMin=-100, yMax=40)
widgets["snrPlot"].setLimits(xMin=-60, xMax=0, yMin=-10, yMax=40)
widgets["snrPlot"].showGrid(True, True)
widgets["snrPlotRange"] = [-10, 30]
widgets["snrPlotTime"] = np.array([])
widgets["snrPlotSNR"] = np.array([])
widgets["snrPlotData"] = widgets["snrPlot"].plot(widgets["snrPlotTime"], widgets["snrPlotSNR"])
widgets["eyeDiagramPlot"] = pg.PlotWidget(title="Eye Diagram")
# TODO: Look into eye diagram more
# widgets["eyeDiagramPlot"] = pg.PlotWidget(title="Eye Diagram")
# widgets["eyeDiagramData"] = widgets["eyeDiagramPlot"].plot([0])
w3_snr.addWidget(widgets["snrPlot"], 0, 1, 2, 1)
w3.addWidget(widgets["snrPlot"], 0, 0)
w3.addWidget(widgets["eyeDiagramPlot"], 0, 1)
#w3.addWidget(widgets["eyeDiagramPlot"], 0, 1)
d2.addWidget(w3)
d2_snr.addWidget(w3_snr)
# Telemetry Data
w4 = pg.LayoutWidget()
@ -349,6 +372,40 @@ def handle_fft_update(data):
widgets["spectrumPlotRange"][0], min(0, _new_max) + 20
)
def handle_status_update(status):
""" Handle a new status frame """
global widgets, habitat
# Update Frequency estimator markers
for _i in range(len(status.extended_stats.f_est)):
_fest_pos = float(status.extended_stats.f_est[_i])
if _fest_pos != 0.0:
widgets["estimatorLines"][_i].setPos(_fest_pos)
# Update SNR Plot
_time = time.time()
# Roll Time/SNR
widgets["snrPlotTime"] = np.append(widgets["snrPlotTime"], _time)
widgets["snrPlotSNR"] = np.append(widgets["snrPlotSNR"], float(status.snr))
if len(widgets["snrPlotTime"]) > 200:
widgets["snrPlotTime"] = widgets["snrPlotTime"][1:]
widgets["snrPlotSNR"] = widgets["snrPlotSNR"][1:]
# Plot new SNR data
widgets["snrPlotData"].setData((widgets["snrPlotTime"]-_time), widgets["snrPlotSNR"])
_old_max = widgets["snrPlotRange"][1]
_tc = 0.1
_new_max = float((_old_max * (1 - _tc)) + (np.max(widgets["snrPlotSNR"]) * _tc))
widgets["snrPlotRange"][1] = _new_max
widgets["snrPlot"].setYRange(
widgets["snrPlotRange"][0], _new_max+10
)
widgets["snrLabel"].setText(f"<b>SNR:</b> {float(status.snr):2.1f} dB")
def add_fft_update(data):
""" Try and insert a new set of FFT data into the update queue """
@ -359,6 +416,37 @@ def add_fft_update(data):
logging.error("FFT Update Queue Full!")
def add_stats_update(frame):
""" Try and insert modem statistics into the processing queue """
global status_update_queue
try:
status_update_queue.put_nowait(frame)
except:
logging.error("Status Update Queue Full!")
def handle_new_packet(frame):
""" Handle receipt of a newly decoded packet """
if len(frame.data) > 0:
if type(frame.data) == bytes:
_packet = frame.data.hex()
else:
_packet = frame.data
widgets["latestSentenceData"].setText(f"{_packet}")
# Immediately upload RTTY packets.
# Why are we getting packets from the decoder twice?
if _packet.startswith('$$$$$'):
habitat_uploader.add(_packet[3:]+'\n')
else:
# TODO: Handle binary packets.
pass
def start_decoding():
global widgets, audio_stream, fft_process, horus_modem, audio_devices, running, fft_update_queue, status_update_queue
@ -368,7 +456,16 @@ def start_decoding():
_sample_rate = int(widgets["audioSampleRateSelector"].currentText())
_dev_index = audio_devices[_dev_name]["index"]
# TODO: Grab horus data here.
# Grab Horus Settings
_modem_name = widgets["horusModemSelector"].currentText()
_modem_id = HORUS_MODEM_LIST[_modem_name]['id']
_modem_rate = int(widgets["horusModemRateSelector"].currentText())
_modem_mask_enabled = widgets["horusMaskEstimatorSelector"].isChecked()
if _modem_mask_enabled:
_modem_tone_spacing = int(widgets["horusMaskSpacingEntry"].text())
else:
_modem_tone_spacing = -1
# Init FFT Processor
NFFT = 2 ** 14
@ -377,7 +474,14 @@ def start_decoding():
nfft=NFFT, stride=STRIDE, fs=_sample_rate, callback=add_fft_update
)
# TODO: Setup modem here
# Setup Modem
horus_modem = HorusLib(
libpath=".",
mode=_modem_id,
rate=_modem_rate,
tone_spacing=_modem_tone_spacing,
callback=handle_new_packet
)
# Setup Audio
audio_stream = AudioStream(
@ -385,7 +489,8 @@ def start_decoding():
fs=_sample_rate,
block_size=fft_process.stride,
fft_input=fft_process.add_samples,
modem=None,
modem=horus_modem,
stats_callback=add_stats_update
)
widgets["startDecodeButton"].setText("Stop")
@ -404,6 +509,11 @@ def start_decoding():
except Exception as e:
logging.exception("Could not stop fft processing.", exc_info=e)
try:
horus_modem.close()
except Exception as e:
logging.exception("Could not close horus modem.", exc_info=e)
fft_update_queue = Queue(256)
status_update_queue = Queue(256)
@ -428,7 +538,8 @@ def processQueues():
while status_update_queue.qsize() > 0:
_status = status_update_queue.get()
# Handle Status updates here.
handle_status_update(_status)
gui_update_timer = QtCore.QTimer()

Wyświetl plik

@ -1,5 +1,6 @@
#!/usr/bin/env python3
import audioop
import ctypes
from ctypes import *
import logging
@ -8,13 +9,6 @@ from enum import Enum
import os
import logging
# TODO
# - Doc Strings
# - frame error checking
# - Modem Stats
# - demodulate should return an object with the stats
MODEM_STATS_NR_MAX = 8
MODEM_STATS_NC_MAX = 50
MODEM_STATS_ET_MAX = 8
@ -24,52 +18,72 @@ MODEM_STATS_MAX_F_EST = 4
class COMP(Structure):
_fields_ = [("real", c_float), ("imag", c_float)]
"""
Used in MODEM_STATS for representing IQ.
"""
_fields_ = [
("real", c_float),
("imag", c_float)
]
class MODEM_STATS(Structure): # modem_stats.h
"""
Extended modem stats structure
"""
_fields_ = [
("Nc", c_int),
("snr_est", c_float),
(
"rx_symbols",
(COMP * MODEM_STATS_NR_MAX) * (MODEM_STATS_NC_MAX + 1),
), # rx_symbols[MODEM_STATS_NR_MAX][MODEM_STATS_NC_MAX+1];
# rx_symbols[MODEM_STATS_NR_MAX][MODEM_STATS_NC_MAX+1];
("rx_symbols", (COMP * MODEM_STATS_NR_MAX)*(MODEM_STATS_NC_MAX+1)),
("nr", c_int),
("sync", c_int),
("foff", c_float),
("rx_timing", c_float),
("clock_offset", c_float),
("sync_metric", c_float),
(
"rx_eye",
(c_float * MODEM_STATS_ET_MAX) * MODEM_STATS_EYE_IND_MAX,
), # float rx_eye[MODEM_STATS_ET_MAX][MODEM_STATS_EYE_IND_MAX];
# float rx_eye[MODEM_STATS_ET_MAX][MODEM_STATS_EYE_IND_MAX];
("rx_eye", (c_float * MODEM_STATS_ET_MAX)*MODEM_STATS_EYE_IND_MAX),
("neyetr", c_int),
("neyesamp", c_int),
("f_est", c_float * MODEM_STATS_MAX_F_EST),
("fft_buf", c_float * 2 * MODEM_STATS_NSPEC),
("fft_cfg", POINTER(c_ubyte)),
("f_est", c_float*MODEM_STATS_MAX_F_EST),
("fft_buf", c_float * 2*MODEM_STATS_NSPEC),
("fft_cfg", POINTER(c_ubyte))
]
class Mode(Enum):
"""
Modes (and aliases for modes) for the HorusLib modem
"""
BINARY = 0
BINARY_V1 = 0
RTTY_7N2 = 99
RTTY_7N2 = 90
RTTY = 90
BINARY_V2_256BIT = 1
BINARY_V2_128BIT = 2
class Frame:
def __init__(
self,
data: bytes,
sync: bool,
crc_pass: bool,
snr: float,
extended_stats: MODEM_STATS,
):
class Frame():
"""
Frame class used for demodulation attempts.
Attributes
----------
data : bytes
Demodulated data output. Empty if demodulation didn't succeed
sync : bool
Modem sync status
snr : float
Estimated SNR
crc_pass : bool
CRC check status
extended_stats
Extended modem stats. These are provided as c_types so will need to be cast prior to use. See MODEM_STATS for structure details
"""
def __init__(self, data: bytes, sync: bool, crc_pass: bool, snr: float, extended_stats: MODEM_STATS):
self.data = data
self.sync = sync
self.snr = snr
@ -77,7 +91,27 @@ class Frame:
self.extended_stats = extended_stats
class HorusLib:
class HorusLib():
"""
HorusLib provides a binding to horuslib to demoulate frames.
Example usage:
from horuslib import HorusLib, Mode
with HorusLib(, mode=Mode.BINARY, verbose=False) as horus:
with open("test.wav", "rb") as f:
while True:
data = f.read(horus.nin*2)
if horus.nin != 0 and data == b'': #detect end of file
break
output = horus.demodulate(data)
if output.crc_pass and output.data:
print(f'{output.data.hex()} SNR: {output.snr}')
for x in range(horus.mfsk):
print(f'F{str(x)}: {float(output.extended_stats.f_est[x])}')
"""
def __init__(
self,
libpath=f"",
@ -86,7 +120,8 @@ class HorusLib:
tone_spacing=-1,
stereo_iq=False,
verbose=False,
callback=None
callback=None,
sample_rate=48000
):
"""
Parameters
@ -94,7 +129,7 @@ class HorusLib:
libpath : str
Path to libhorus
mode : Mode
horuslib.Mode.BINARY, horuslib.Mode.BINARY_V2_256BIT, horuslib.Mode.BINARY_V2_128BIT, horuslib.Mode.RTTY_7N2
horuslib.Mode.BINARY, horuslib.Mode.BINARY_V2_256BIT, horuslib.Mode.BINARY_V2_128BIT, horuslib.Mode.RTTY, RTTY_7N2 = 99
rate : int
Changes the modem rate for supported modems. -1 for default
tone_spacing : int
@ -104,7 +139,9 @@ class HorusLib:
verbose : bool
Enabled horus_set_verbose
callback : function
Callback function to run on packet detection.
When set you can use add_samples to add any number of audio frames and callback will be called when a demodulated frame is avaliable.
sample_rate : int
The input sample rate of the audio input
"""
if sys.platform == "darwin":
@ -114,9 +151,8 @@ class HorusLib:
else:
libpath = os.path.join(libpath, "libhorus.so")
self.c_lib = ctypes.cdll.LoadLibrary(
libpath
) # future improvement would be to try a few places / names
# future improvement would be to try a few places / names
self.c_lib = ctypes.cdll.LoadLibrary(libpath)
# horus_open_advanced
self.c_lib.horus_open_advanced.restype = POINTER(c_ubyte)
@ -155,8 +191,6 @@ class HorusLib:
# horus_rx
self.c_lib.horus_rx.restype = c_int
# struct horus *hstates, char ascii_out[], short demod_in[], int quadrature
if type(mode) != type(Mode(0)):
raise ValueError("Must be of type horuslib.Mode")
else:
@ -198,6 +232,10 @@ class HorusLib:
self.mfsk = int(self.c_lib.horus_get_mFSK(self.hstates))
self.resampler_state = None
self.audio_sample_rate = sample_rate
self.modem_sample_rate = 48000
# in case someone wanted to use `with` style. I'm not sure if closing the modem does a lot.
def __enter__(self):
return self
@ -205,17 +243,34 @@ class HorusLib:
def __exit__(self, *a):
self.close()
def close(self):
def close(self) -> None:
"""
Closes Horus modem.
"""
self.c_lib.horus_close(self.hstates)
logging.debug("Shutdown horus modem")
def update_nin(self):
def _update_nin(self) -> None:
"""
Updates nin. Called every time RF is demodulated and doesn't need to be run manually
"""
new_nin = int(self.c_lib.horus_nin(self.hstates))
if self.nin != new_nin:
logging.debug(f"Updated nin {new_nin}")
self.nin = new_nin
def demodulate(self, demod_in: bytes):
def demodulate(self, demod_in: bytes) -> Frame:
"""
Demodulates audio in, into bytes output.
Parameters
----------
demod_in : bytes
16bit, signed for audio in. You'll need .nin frames in to work correctly.
"""
# resample to 48khz
(demod_in, self.resampler_state) = audioop.ratecv(demod_in, 2, 1+int(self.stereo_iq), self.audio_sample_rate, self.modem_sample_rate, self.resampler_state)
# from_buffer_copy requires exact size so we pad it out.
buffer = bytearray(
len(self.DemodIn()) * sizeof(c_short)
@ -237,7 +292,7 @@ class HorusLib:
crc = bool(self.c_lib.horus_crc_ok(self.hstates))
data_out = bytes(data_out)
self.update_nin()
self._update_nin()
# strip the null terminator out
data_out = data_out[:-1]
@ -251,10 +306,13 @@ class HorusLib:
data_out = bytes.fromhex(data_out.decode("ascii"))
except ValueError:
logging.debug(data_out)
logging.error("💥Couldn't decode the hex from the modem")
logging.error("Couldn't decode the hex from the modem")
return bytes()
else:
data_out = bytes(data_out.decode("ascii"))
# Ascii
data_out = data_out.decode("ascii")
# Strip of all null characters.
data_out = data_out.rstrip('\x00')
frame = Frame(
data=data_out,
@ -265,11 +323,9 @@ class HorusLib:
)
return frame
def add_samples(self, samples: bytes, rate: int=48000):
def add_samples(self, samples: bytes):
""" Add samples to a input buffer, to pass on to demodulate when we have nin samples """
# TODO: Resampling support
# Add samples to input buffer
self.input_buffer.extend(samples)
@ -277,7 +333,7 @@ class HorusLib:
_frame = None
while _processing:
# Process data until we have less than _nin samples.
_nin = self.nin
_nin = int(self.nin*(self.audio_sample_rate/self.modem_sample_rate))
if len(self.input_buffer) > (_nin * 2):
# Demodulate
_frame = self.demodulate(self.input_buffer[:(_nin*2)])
@ -297,7 +353,8 @@ class HorusLib:
if __name__ == "__main__":
import sys
if len(sys.argv) != 2:
raise ArgumentError("Usage python3 -m horuslib filename")
filename = sys.argv[1]
def frame_callback(frame):
@ -307,20 +364,8 @@ if __name__ == "__main__":
logging.basicConfig(
format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG
)
# with HorusLib(libpath=".", mode=Mode.BINARY, verbose=False) as horus:
# with open(filename, "rb") as f:
# while True:
# data = f.read(horus.nin * 2)
# if horus.nin != 0 and data == b"": # detect end of file
# break
# output = horus.demodulate(data)
# if output.crc_pass and output.data:
# print(f"{output.data.hex()} SNR: {output.snr}")
# for x in range(horus.mfsk):
# print(f"F{str(x)}: {float(output.extended_stats.f_est[x])}")
with HorusLib(libpath=".", mode=Mode.BINARY, verbose=False, callback=frame_callback) as horus:
with HorusLib(libpath=".", mode=Mode.BINARY, verbose=False, callback=frame_callback, sample_rate=8000) as horus:
with open(filename, "rb") as f:
while True:
# Fixed read size - 2000 samples
@ -329,4 +374,4 @@ if __name__ == "__main__":
break
output = horus.add_samples(data)
if output:
print(f"Sync: {output.sync} SNR: {output.snr}")
print(f"Sync: {output.sync} SNR: {output.snr}")

Wyświetl plik

@ -1,18 +1,19 @@
# Modem Interfacing
import logging
from .horuslib import Mode
# Modem paramers and defaults
HORUS_MODEM_LIST = {
"Horus Binary v1 (Legacy)": {
"id": 0,
"id": Mode.BINARY_V1,
"baud_rates": [50, 100, 300],
"default_baud_rate": 100,
"default_tone_spacing": 270,
"use_mask_estimator": False,
},
"RTTY (7N2)": {
"id": 99,
"id": Mode.RTTY_7N2,
"baud_rates": [50, 100, 300, 600, 1000],
"default_baud_rate": 100,
"default_tone_spacing": 425,