From 558e4cf7022521da3e57254ab62d4dff2d9468f9 Mon Sep 17 00:00:00 2001 From: sm3ulc Date: Tue, 19 Jan 2021 21:05:22 +0100 Subject: [PATCH] new wsprnet format --- balloon.ini | 15 +++- sonde_to_aprs.py | 191 ++++++++++++++++------------------------------- sonde_to_html.py | 100 +++++++++++++++++++++++++ telemetry.py | 33 +++++--- webscrape.py | 18 ++--- 5 files changed, 204 insertions(+), 153 deletions(-) create mode 100644 sonde_to_html.py diff --git a/balloon.ini b/balloon.ini index 0b5659f..ff623bc 100644 --- a/balloon.ini +++ b/balloon.ini @@ -6,13 +6,22 @@ push_aprs = True # This is the callsign the object comes from. This should NOT have any SSID's on it (i.e. -9) aprsCallsign = "myhamcall" + # Replace with your own callsign. This should NOT have any SSID's on it (i.e. -9) aprsUser = 'myaprsisuser' + # APRS-IS passcode for your callsign. aprsPass = 'myaprsispass' -# [ habhub name, aprs-call, band in mhz, channel, timeslot ] -balloons = [[ "BSY22","SA7XXX",14,11,0 ], - [ "B2", "SOMECALL",14,12,0 ]] +push_html = False +push_ftp = False +ftp_server = ftp.uus.ro +ftp_username = xxx +ftp_password = xxx + +# [ habhub name, aprs-wspr-call, band in mhz, channel, timeslot, datetime, html_push, aprs-ssid, aprs_comment] :: html_push ONLY for one balloon + +balloons = [["BS234","CALL1",14,11,0,"20200831T0557",0,"12","Balloon 20m WSPR"], + ["ISDF","CALL2",14,15,2,"20200831T0558",0,"12","Balloon 20m WSPR"]] diff --git a/sonde_to_aprs.py b/sonde_to_aprs.py index 00dcacf..38c595a 100755 --- a/sonde_to_aprs.py +++ b/sonde_to_aprs.py @@ -20,6 +20,7 @@ import configparser import time, datetime, urllib3, sys +import logging from socket import * # APRS-IS login info @@ -35,144 +36,78 @@ aprsPass = config['main']['aprsPass'] # This is the callsign the object comes from. Doesn't necessarily have to be the same as your APRS-IS login. callsign = config['main']['aprsCallsign'] -# Get KML from SondeMonitor and parse into a Python dictionary def get_sonde(): - sonde_data = {} - sonde_data["lat"] = str(sys.argv[2]) - sonde_data["lon"] = str(sys.argv[3]) - sonde_data["alt"] = "10000" - sonde_data["id"] = str(sys.argv[1]) - return sonde_data + sonde_data = {} + sonde_data["lat"] = str(sys.argv[2]) + sonde_data["lon"] = str(sys.argv[3]) + sonde_data["alt"] = "10000" + sonde_data["id"] = str(sys.argv[1]) + sonde_data["speed"] = "0" + sonde_data["temp"] = "0" + sonde_data["batt"] = "0" + sonde_data["comment"] = str(sys.argv[8]) + return sonde_data # Push a Radiosonde data packet to APRS as an object. def push_balloon_to_aprs(sonde_data): - # Pad or limit the sonde ID to 9 characters. - object_name = sonde_data["id"] - if len(object_name) > 9: - object_name = object_name[:9] - elif len(object_name) < 9: - object_name = object_name + " "*(9-len(object_name)) + # Pad or limit the sonde ID to 9 characters. + object_name = sonde_data["id"] + if len(object_name) > 9: + object_name = object_name[:9] + elif len(object_name) < 9: + object_name = object_name + " "*(9-len(object_name)) - # Convert float latitude to APRS format (DDMM.MM) - lat = float(sonde_data["lat"]) - lat_degree = abs(int(lat)) - lat_minute = abs(lat - int(lat)) * 60.0 - lat_min_str = ("%02.2f" % lat_minute).zfill(5) - lat_dir = "S" - if lat>0.0: - lat_dir = "N" - lat_str = "%02d%s" % (lat_degree,lat_min_str) + lat_dir + # Convert float latitude to APRS format (DDMM.MM) + lat = float(sonde_data["lat"]) + lat_degree = abs(int(lat)) + lat_minute = abs(lat - int(lat)) * 60.0 + lat_min_str = ("%02.2f" % lat_minute).zfill(5) + lat_dir = "S" + if lat>0.0: + lat_dir = "N" + + lat_str = "%02d%s" % (lat_degree,lat_min_str) + lat_dir - # Convert float longitude to APRS format (DDDMM.MM) - lon = float(sonde_data["lon"]) - lon_degree = abs(int(lon)) - lon_minute = abs(lon - int(lon)) * 60.0 - lon_min_str = ("%02.2f" % lon_minute).zfill(5) - lon_dir = "E" - if lon<0.0: - lon_dir = "W" - lon_str = "%03d%s" % (lon_degree,lon_min_str) + lon_dir - - # Convert Alt (in metres) to feet - alt = int(float(sonde_data["alt"])/0.3048) - - # Produce the APRS object string. - out_str = ";%s*111111z%s/%sO000/000/A=%06d Balloon" % (object_name,lat_str,lon_str,alt) - print(out_str) + # Convert float longitude to APRS format (DDDMM.MM) + lon = float(sonde_data["lon"]) + lon_degree = abs(int(lon)) + lon_minute = abs(lon - int(lon)) * 60.0 + lon_min_str = ("%02.2f" % lon_minute).zfill(5) + lon_dir = "E" + if lon<0.0: + lon_dir = "W" + lon_str = "%03d%s" % (lon_degree,lon_min_str) + lon_dir - # Connect to an APRS-IS server, login, then push our object position in. - + # Convert Alt (in metres) to feet + alt = int(float(sonde_data["alt"])/0.3048) + + # Convert Speed (in metres) to feet + speed = round(float(sonde_data["speed"])) + + #if speed < 1: + # speed = 1; + + temp = round(float(sonde_data["temp"]),1) + batt = round(float(sonde_data["batt"]),2) + object_comment = sonde_data["comment"] + + + # Produce the APRS object string. + #out_str = ";%s*111111z%s/%sO000/000/A=%06d Balloon" % (object_name,lat_str,lon_str,alt) + # print(out_str) + out_str = ";%s*111111z%s/%sO000/%03d/A=%06dTemp=%sC Solar=%sV %s" % (object_name,lat_str,lon_str,speed,alt,temp,batt,object_comment) + logging.info('\033[33m' + "APRS: %s" + '\033[0m' , out_str) + + # Connect to an APRS-IS server, login, then push our object position in. # create socket & connect to server - sSock = socket(AF_INET, SOCK_STREAM) - sSock.connect((serverHost, serverPort)) + sSock = socket(AF_INET, SOCK_STREAM) + sSock.connect((serverHost, serverPort)) # logon - sSock.send(b'user %s pass %s vers VK5QI-Python 0.01\n' % (aprsUser.encode('utf-8'), aprsPass.encode('utf-8')) ) + sSock.send(b'user %s pass %s vers VK5QI-Python 0.01\n' % (aprsUser.encode('utf-8'), aprsPass.encode('utf-8')) ) # send packet - sSock.send(b'%s>APRS:%s\n' % (callsign.encode('utf-8'), out_str.encode('utf-8')) ) + sSock.send(b'%s>APRS:%s\n' % (callsign.encode('utf-8'), out_str.encode('utf-8')) ) # close socket - sSock.shutdown(0) - sSock.close() + sSock.shutdown(0) + sSock.close() - - -# VE3OCL-11:PARM.Speed,Temp,Vbat,GPS,Sats -# VE3OCL-11:UNIT.kn,C,V,, -# VE3OCL-11:BITS.11111111,10mW research balloon -# VE3OCL-11:EQNS.0,0.1,0,0,0.1,-273.2,0,0.001,0,0,1,0,0,1,0 - -# $speed = int(($speed * 10) + 0.5); -# $temp = int((($temp + 273.2) * 10) + 0.5); -# $vbat = int(($vbat * 1000) + 0.5); - - - - - -# For explanation of encoding see: -# http://he.fi/doc/aprs-base91-comment-telemetry.txt - - -# sub usage () -# { -# print STDERR "\n"; -# print STDERR "telem-data91.pl - Format data into compressed base 91 telemetry.\n"; -# print STDERR "\n"; -# print STDERR "Usage: telem-data91.pl sequence value1 [ value2 ... ]\n"; -# print STDERR "\n"; -# print STDERR "A sequence number and up to 5 analog values can be specified.\n"; -# print STDERR "Any sixth value must be 8 binary digits.\n"; -# print STDERR "Values must be integers in range of 0 to 8280.\n"; - - -# if ($#ARGV+1 < 2 || $#ARGV+1 > 7) { -# print STDERR "2 to 7 command line arguments must be provided.\n"; -# usage(); -# } - - -# if ($#ARGV+1 == 7) { -# if ( ! ($ARGV[6] =~ m/^[01]{8}$/)) { -# print STDERR "The sixth value must be 8 binary digits.\n"; -# usage(); -# } -# # Convert binary digits to value. -# $ARGV[6] = oct("0b" . reverse($ARGV[6])); -# } - -# $result = "|"; - -# for ($n = 0 ; $n <= $#ARGV; $n++) { -# #print $n . " = " . $ARGV[$n] . "\n"; -# $v = $ARGV[$n]; -# if ($v != int($v) || $v < 0 || $v > 8280) { -# print STDERR "argn $n - $v is not an integer in range of 0 to 8280.\n"; -# usage(); -# } - -# $result .= base91($v); -# } - -# $result .= "|"; -# print "$result\n"; -# exit 0; - - -# sub base91 () -# { -# my $x = @_[0]; - -# my $d1 = int ($x / 91); -# my $d2 = $x % 91; - -# return chr($d1+33) . chr($d2+33); -# } -# : - -# Py2 & Py3 compability -# import sys -# if sys.version_info[0] >= 3: -# is_py3 = True -# string_type = (str, ) -# string_type_parse = string_type + (bytes, ) -# int_type = int diff --git a/sonde_to_html.py b/sonde_to_html.py new file mode 100644 index 0000000..198ba80 --- /dev/null +++ b/sonde_to_html.py @@ -0,0 +1,100 @@ +import fileinput +import sys +import ftplib +import datetime +import configparser +import ftplib +import json + +config = configparser.ConfigParser( + converters = { + 'datetime': lambda s : datetime.datetime.fromisoformat(s) + } +) +config.read('balloon.ini') + +push_ftp = config['main'].getboolean('push_ftp') +ftp_server = config['main']['ftp_server'] +ftp_username = config['main']['ftp_username'] +ftp_password = config['main']['ftp_password'] +balloons = json.loads(config.get('main','balloons')) +then = datetime.datetime.strptime(balloons[0][5], "%Y%m%dT%H%M") + +def getDuration(then, now, interval = "default"): + + # Returns a duration as specified by variable interval + # Functions, except totalDuration, returns [quotient, remainder] + + duration = now - then # For build-in functions + duration_in_s = duration.total_seconds() + + def months(): + return duration_in_s // 2592000 # Seconds in a month=2592000. + + def days(): + return (duration_in_s // 86400 - 30 * months()) % 30# Seconds in a day = 86400 + + def hours(): + return (duration_in_s // 3600 - 24 * days()) % 24# Seconds in an hour = 3600 + + def minutes(): + return (duration_in_s // 60 - 60 * hours()) % 60# Seconds in a minute = 60 + + return { + 'months': int(months()), + 'days': int(days()), + 'hours': int(hours()), + 'minutes': int(minutes()), + }[interval] + +def push_balloon_to_html(telemetry): + def add_position(file): # search and replace for the map position + searchExp = "// #POSITION#" + replaceExp = f"\t\tnew google.maps.LatLng({telemetry['lat']},{telemetry['lon']}),\n// #POSITION#" + for line in fileinput.input(file, inplace=1): + if searchExp in line: + line = line.replace(searchExp,replaceExp) + sys.stdout.write(line) + add_position('ICT_RPI.html'); + + + def update_popup1(file): # search and replace for the telemetry popup1 + time = telemetry['time'].strftime("%d-%b-%Y %H%M") + time_now = datetime.datetime.utcnow() + time_now_delta = time_now.strftime("%Y%m%dT%H%M") + #print(then) + #print(telemetry['time']) + + searchExp = "'

Updated" + replaceExp = f"'

Updated {time}Z
Locator = {telemetry['loc'].upper()}
Duration = {getDuration(then,telemetry['time'],'months')}mo {getDuration(then,telemetry['time'],'days')}d {getDuration(then,telemetry['time'],'hours')}h {getDuration(then,telemetry['time'],'minutes')}m
Distance = '+ distance + \n" + for line in fileinput.input(file, inplace=1): + if searchExp in line: + line = line.replace(line,replaceExp) + sys.stdout.write(line) + update_popup1('ICT_RPI.html'); + + def update_popup2(file): # search and replace for the telemetry popup2 + searchExp = "'km
Altitude" + replaceExp = f"'km
Altitude = {telemetry['alt']}m
Speed = {telemetry['speed']}kt {round(telemetry['speed']*1.852)}km/h
Solar = {round(telemetry['batt'],2)}V, Temp = {round(telemetry['temp'],1)}C
GPS = {int(telemetry['gps'])}, Sats = {int(telemetry['sats'])}

'; \n" + for line in fileinput.input(file, inplace=1): + if searchExp in line: + line = line.replace(line,replaceExp) + sys.stdout.write(line) + update_popup2('ICT_RPI.html'); + + def update_txt(): # add telemetry to the txt file + telestr = "Telemetry ICT6: %s,%s,%d,%d,%.1f,%.2f,%d,%d\n" % (telemetry['time'].strftime("%d-%b-%Y %H%M"), telemetry['loc'], telemetry['alt'], round(telemetry['speed']*1.852), telemetry['temp'], telemetry['batt'], telemetry['gps'], telemetry['sats']) + with open("Output.txt", "a") as text_file: + text_file.write(telestr) + + update_txt(); + + if push_ftp: + session = ftplib.FTP(ftp_server,ftp_username,ftp_password) + file = open('ICT_RPI.html','rb') # file to send + session.storbinary('STOR flights/ICT6.html', file) # send the file + file.close() # close file and FTP + file = open('Output.txt','rb') # file to send + session.storbinary('STOR flights/ICT6_telemetry.txt', file) # send the file + file.close() # close file and FTP + session.quit() \ No newline at end of file diff --git a/telemetry.py b/telemetry.py index 7d81f7f..f07b0eb 100644 --- a/telemetry.py +++ b/telemetry.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3.6 +#!/usr/bin/env python3 from base64 import b64encode import configparser @@ -22,6 +22,7 @@ import maidenhead from balloon import * from sonde_to_aprs import * +from sonde_to_html import * # Power to decixmal conversion table pow2dec = {0:0,3:1,7:2,10:3,13:4,17:5,20:6,23:7,27:8,30:9,33:10,37:11,40:12,43:13,47:14,50:15,53:16,57:17,60:18} @@ -140,7 +141,7 @@ def readgz(balloons, gzfile): # print("Found", c, row) spots.append(row) - if re.match('(^0|^Q).[0-9].*', row[1]): + if re.match('(^0|^1|^Q).[0-9].*', row[1]): row[0] = datetime.datetime.fromtimestamp(int(row[0])) row[3] = int(row[3]) @@ -398,11 +399,11 @@ def send_tlm_to_habitat(sentence, callsign, spot_time): body=json.dumps(data), ) - print(resp['status']) + # print(resp['status']) if resp['status'] == '201': - print("OK 201", input) + logging.info("OK 201") elif resp['status'] == '403': - print("Error 403 - already uploaded") + logging.info("Error 403 - already uploaded") return @@ -436,13 +437,13 @@ def timetrim(spots, m): # # Main function - filter, process and upload of telemetry # -def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs): +def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html): # Filter out telemetry-packets spots_tele = [] for row in spots: # print(row) - if re.match('(^0|^Q).[0-9].*', row[1]): + if re.match('(^0|^1|^Q).[0-9].*', row[1]): # print(', '.join(row)) #if re.match('10\..*', row[2]) or re.match('14\..*', row[2]): spots_tele.append(row) @@ -460,7 +461,10 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs): balloon_mhz = b[2] balloon_channel = b[3] balloon_timeslot = b[4] - + balloon_html_push = b[6] + balloon_ssid = b[7] + balloon_aprs_comment = b[8] + logging.info("Name: %-8s Call: %6s MHz: %2d Channel: %2d Slot: %d" % (balloon_name, balloon_call, balloon_mhz, balloon_channel, balloon_timeslot)) # Filter out telemetry for active channel @@ -581,7 +585,6 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs): # push_habhub = True if push_habhub: # Send telemetry to habhub - # send_tlm_to_habitat2(telestr, habhub_callsign) send_tlm_to_habitat(telestr, habhub_callsign, spot_time) else: logging.info("Not pushing to habhub") @@ -589,15 +592,25 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs): if push_aprs: # Prep and push basic data to aprs.fi sonde_data = {} - sonde_data["id"] = spot_call + "-12" + sonde_data["id"] = spot_call + "-" + balloon_ssid sonde_data["lat"] = telemetry['lat'] sonde_data["lon"] = telemetry['lon'] sonde_data["alt"] = telemetry['alt'] + sonde_data["speed"] = telemetry['speed'] + sonde_data["temp"] = telemetry['temp'] + sonde_data["batt"] = telemetry['batt'] + sonde_data["comment"] = balloon_aprs_comment logging.info("Pushing data to aprs.fi") push_balloon_to_aprs(sonde_data) else: logging.info("Not pushing to aprs.fi") + if push_html and balloon_html_push: + # Push basic data to ftp html + logging.info('\033[33m' + "Pushing data to html page" + '\033[0m') + push_balloon_to_html(telemetry) + + # Add sent string to history-db addsentdb(balloon_name, row[0], telestr) diff --git a/webscrape.py b/webscrape.py index 9ef6076..36a42e2 100755 --- a/webscrape.py +++ b/webscrape.py @@ -102,7 +102,7 @@ def balloonfilter(spots,balloons): row[5] = row[5][0:4] filtered.append(row) - if re.match('(^0|^Q).[0-9].*', row[1]): + if re.match('(^0|^1|^Q).[0-9].*', row[1]): filtered.append(row) # for r in filtered: @@ -178,9 +178,9 @@ for opt, arg in options: config = configparser.ConfigParser() config.read(conf_file) -push_habhub = config['main']['push_habhub'] -push_aprs = config['main']['push_aprs'] - +push_habhub = config['main'].getboolean('push_habhub') +push_aprs = config['main'].getboolean('push_aprs') +push_html = config['main'].getboolean('push_html') balloons = json.loads(config.get('main','balloons')) @@ -254,7 +254,7 @@ if csv_file: sys.exit(0) # Spots to pullfrom wsprnet -nrspots_pull= 2000 +nrspots_pull= 3000 spotcache = [] logging.info("Preloading cache from wsprnet...") @@ -322,10 +322,9 @@ while 1==1: # Filter out all spots newer that x minutes spots = timetrim(spots,60) - if len(spots) > 1: logging.info("pre-tele: %d",len(spots)) - spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs) + spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html) logging.info("pro-tele: %s", str(len(spots))) if new_max < len(newspots): @@ -336,11 +335,6 @@ while 1==1: logging.info("Hit max spots. Increasing set to fetch") nrspots_pull += 100 - # print("%s Spots: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d Hitrate: %5.2f%%" % - # (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(spotcache), len(newspots), new_max, nrspots_pull, str(datetime.datetime.now() - tnow).split(":")[2], src_cc, 100-(src_cc / (len(spotcache)*nrspots_pull))*100)) - - # print("%s Spots: %5d Cache: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d" % - # (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), len(spots), len(spotcache), len(newspots), new_max, nrspots_pull, str(datetime.datetime.now() - tnow).split(":")[2], src_cc)) printstr = ("Spots: %5d Cache: %6d New: %5d (max: %5d) Nrspots: %5d Looptime: %s Checks: %8d" % (len(spots), len(spotcache), len(newspots), new_max, nrspots_pull, str(datetime.datetime.now() - tnow).split(":")[2], src_cc)) logging.info(printstr)