From af870d89e03ad76595b55e82f9a1de6a2ef06027 Mon Sep 17 00:00:00 2001 From: Sven Steudte Date: Sun, 8 Oct 2017 19:29:17 +0200 Subject: [PATCH] Split image packets in half, fixed timezone bug --- decoder/decoder.py | 11 +- decoder/html/index.php | 15 +-- decoder/image.py | 186 ++++++++++++++++++++----------- decoder/position.py | 22 ++-- tracker/software/config.c | 4 +- tracker/software/threads/image.c | 22 +++- 6 files changed, 164 insertions(+), 96 deletions(-) diff --git a/decoder/decoder.py b/decoder/decoder.py index f58cc42..2d5ea8d 100755 --- a/decoder/decoder.py +++ b/decoder/decoder.py @@ -30,6 +30,7 @@ sqlite.cursor().execute(""" lat FLOAT, lon FLOAT, alt INTEGER, + new INTEGER, comment TEXT, sequ INTEGER, tel1 INTEGER, @@ -50,7 +51,9 @@ sqlite.cursor().execute(""" lat FLOAT, lon FLOAT, alt INTEGER, - data TEXT, + data1 TEXT, + data2 TEXT, + crc TEXT, PRIMARY KEY (call, time, imageID, packetID) ) """) @@ -70,7 +73,7 @@ def received_data(data): all = re.search("(.*)\>APECAN(.*?):\/([0-9]{6}h)(.{13})", data) pos = re.search("(.*)\>APECAN(.*?):\/([0-9]{6}h)(.{13})(.*?)\|(.*)\|", data) - dat = re.search("(.*)\>APECAN(.*?):\/([0-9]{6}h)(.{13})(I|L)(.*)", data) + dat = re.search("(.*)\>APECAN(.*?):\/([0-9]{6}h)(.{13})(I|J|L)(.*)", data) if all: call = all.group(1) @@ -90,8 +93,8 @@ def received_data(data): data_b91 = dat.group(6) data = base91.decode(data_b91) # Decode Base91 - if typ is 'I': # Image packet - image.insert_image(sqlite, rxer, call, tim, posi, data, args.server, args.grouping) + if typ is 'I' or typ is 'J': # Image packet + image.insert_image(sqlite, rxer, call, tim, posi, data, typ, args.server, args.grouping) elif typ is 'L': # Log packet position.insert_log(sqlite, call, data) diff --git a/decoder/html/index.php b/decoder/html/index.php index 59718e4..381605f 100644 --- a/decoder/html/index.php +++ b/decoder/html/index.php @@ -85,8 +85,8 @@ function drawItems() { } function initMap() { map = new google.maps.Map(document.getElementById('map'), { - zoom: 9, - center: new google.maps.LatLng(53.2,7.1), + zoom: 12, + center: new google.maps.LatLng(52.45,13.5), gestureHandling: 'greedy' }); @@ -98,14 +98,3 @@ function initMap() { - - - - - - - - - - - diff --git a/decoder/image.py b/decoder/image.py index 17f2d9f..850ee4e 100644 --- a/decoder/image.py +++ b/decoder/image.py @@ -38,88 +38,146 @@ reciever - The call of the receiving station call - The call of the transmitter data - Binary data """ -def insert_image(sqlite, receiver, call, tim, posi, data, server, grouping): +def insert_image(sqlite, receiver, call, tim, posi, data, typ, server, grouping): global jsons - if len(data) != 214: + if len(data) != 110: return # APRS message sampled too short - # Encode callsign (ensure callsign has no more than 6 chars) - bcall = call.split('-') # Split callsign and SSID - if len(bcall) == 1: # No SSID available, so take the callsign - bcall = bcall[0][0:6] - elif(len(bcall[0]) < 5): # Callsign has 4 chars, so take it with the SSID - bcall = bcall[0] + bcall[1][0:2] - elif(len(bcall[0]) < 6): # Callsign has 5 chars, so take it with the last digit of the SSID - bcall = bcall[0] + bcall[1][-1] - else: - bcall = bcall[0][0:6] # Callsign has 6 chars, so take the call without SSID - - data = binascii.unhexlify('66%08x' % encode_callsign(bcall)) + data - - # Calculate CRC for SSDV server - crc = binascii.crc32(data) & 0xffffffff - - # Create message for SSDV server (and save to array) - ssdv = '55' + binascii.hexlify(data).decode("ascii") + ('%08x' % crc) + (64*'0') - jsons.append("""{ - \"type\": \"packet\", - \"packet\": \"""" + ssdv + """\", - \"encoding\": \"hex\", - \"received\": \"""" + datetime.datetime.now().isoformat('T')[:19] + """Z\", - \"receiver\": \"""" + receiver + """\" - }""") - # Decode various meta data - timd,x,y,z,dummy = decode_position(tim, posi, None) - imageID = data[5] - packetID = (data[6] << 8) | data[7] + timd,x,y,z,teld,new = decode_position(tim, posi, None) + imageID = data[0] + packetID = (data[1] << 8) | data[2] + data = binascii.hexlify(data[3:]).decode("ascii") + print(len(data)) # Debug print('Received packet from %s image %d packet %d' % (call, imageID, packetID)) # Insert sqlite.cursor().execute(""" - INSERT OR REPLACE INTO image (call,time,imageID,packetID,lat,lon,alt,data) - VALUES (?,?,?,?,?,?,?,?)""", - (call, int(timd.timestamp()), imageID, packetID, y, x, int(z), ssdv) + INSERT OR IGNORE INTO image (call,time,imageID,packetID,lat,lon,alt) + VALUES (?,?,?,?,?,?,?)""", + (call, int(timd.timestamp()), imageID, packetID, y, x, int(z)) ) sqlite.commit() - # SSDV decode + if typ is 'I': + + # Encode callsign (ensure callsign has no more than 6 chars) + bcall = call.split('-') # Split callsign and SSID + if len(bcall) == 1: # No SSID available, so take the callsign + bcall = bcall[0][0:6] + elif(len(bcall[0]) < 5): # Callsign has 4 chars, so take it with the SSID + bcall = bcall[0] + bcall[1][0:2] + elif(len(bcall[0]) < 6): # Callsign has 5 chars, so take it with the last digit of the SSID + bcall = bcall[0] + bcall[1][-1] + else: + bcall = bcall[0][0:6] # Callsign has 6 chars, so take the call without SSID + + data = '66%08x' % encode_callsign(bcall) + data + + sqlite.cursor().execute(""" + UPDATE image + SET data1 = ? + WHERE call = ? + AND time = ? + AND imageID = ? + AND packetID = ?""", + (data, call, int(timd.timestamp()), imageID, packetID) + ) + + elif typ is 'J': + + sqlite.cursor().execute(""" + UPDATE image + SET data2 = ? + WHERE call = ? + AND time = ? + AND imageID = ? + AND packetID = ?""", + (data, call, int(timd.timestamp()), imageID, packetID) + ) + + sqlite.commit() + + + # Get both data entries cur = sqlite.cursor() - cur.execute("SELECT GROUP_CONCAT(data,'') FROM image WHERE call = ? AND imageID = ? AND time = ? GROUP BY imageID ORDER BY packetID", (call, imageID, int(timd.timestamp()))) - name = 'html/images/%s-%d-%d.jpg' % (call.replace('-',''), int(timd.timestamp()), imageID) - f = open(name, 'wb') - process = Popen(['./ssdv', '-d'], stdin=PIPE, stdout=f, stderr=PIPE) - process.stdin.write(binascii.unhexlify(cur.fetchall()[0][0])) - dummy,err = process.communicate() - f.close() + cur.execute(""" + SELECT data1,data2 + FROM image + WHERE call = ? + AND time = ? + AND imageID = ? + AND packetID = ?""", + (call, int(timd.timestamp()), imageID, packetID) + ) + row = cur.fetchall()[0] - if len(jsons) >= grouping: # Enough packets collected, send them all to the server + if row[0] != None and row[1] != None: # Both entries have been received - req = urllib.request.Request(server) - req.add_header('Content-Type', 'application/json') + data = ''.join(row) - json = "{\"type\":\"packets\",\"packets\":[" + ",".join(jsons) + "]}" # Group all SSDV packets into a big JSON - jsons = [] + # Calculate CRC for SSDV server + crc = binascii.crc32(binascii.unhexlify(data)) & 0xffffffff - try: - error = True - while error: - print('Send to SSDV data server') - try: - result = urllib.request.urlopen(req, "".join(json.split(' ')).encode("ascii")) # Send packets to server - print('Response from Server: OK') - error = False - except urllib.error.URLError as error: - if error.code == 400: - print('Response from Server: %s', error.read()) - error = False - else: - print('SSDV-Server connection error... try again') + # Update CRC in DB + sqlite.cursor().execute(""" + UPDATE image + SET crc = ? + WHERE call = ? + AND time = ? + AND imageID = ? + AND packetID = ?""", + ("%08x" % crc, call, int(timd.timestamp()), imageID, packetID) + ) + sqlite.commit() - except urllib.error.HTTPError as error: # The server did not like our packets :( - print('Send to SSDV data server: failed (the server did not like our packets :( )') - print(error.read()) + # SSDV decode + cur = sqlite.cursor() + cur.execute("SELECT GROUP_CONCAT('55' || data1 || data2 || crc || '0000000000000000000000000000000000000000000000000000000000000000', '') FROM image WHERE call = ? AND imageID = ? AND time = ? GROUP BY imageID ORDER BY packetID", (call, imageID, int(timd.timestamp()))) + name = 'html/images/%s-%d-%d.jpg' % (call.replace('-',''), int(timd.timestamp()), imageID) + f = open(name, 'wb') + process = Popen(['./ssdv', '-d'], stdin=PIPE, stdout=f, stderr=PIPE) + process.stdin.write(binascii.unhexlify(cur.fetchall()[0][0])) + dummy,err = process.communicate() + f.close() + + # Create message for SSDV server (and save to array) + ssdv = '55' + data + ('%08x' % crc) + (64*'0') + jsons.append("""{ + \"type\": \"packet\", + \"packet\": \"""" + ssdv + """\", + \"encoding\": \"hex\", + \"received\": \"""" + datetime.datetime.now().isoformat('T')[:19] + """Z\", + \"receiver\": \"""" + receiver + """\" + }""") + + if len(jsons) >= grouping: # Enough packets collected, send them all to the server + + req = urllib.request.Request(server) + req.add_header('Content-Type', 'application/json') + + json = "{\"type\":\"packets\",\"packets\":[" + ",".join(jsons) + "]}" # Group all SSDV packets into a big JSON + jsons = [] + + try: + err = True + while err: + print('Send to SSDV data server') + try: + result = urllib.request.urlopen(req, "".join(json.split(' ')).encode("ascii")) # Send packets to server + print('Response from Server: OK') + err = False + except urllib.error.URLError as error: + if error.code == 400: + print('Response from Server: %s', error.read()) + err = False + else: + print('SSDV-Server connection error... try again') + + except urllib.error.HTTPError as error: # The server did not like our packets :( + print('Send to SSDV data server: failed (the server did not like our packets :( )') + print(error.read()) diff --git a/decoder/position.py b/decoder/position.py index 808d0b9..be099fb 100644 --- a/decoder/position.py +++ b/decoder/position.py @@ -49,11 +49,15 @@ def decode_position(tim, posi, tel): z = pow(1.002, ze) / 3.281 # Decode time - timd = datetime.now() + timd = datetime.now(timezone.utc) timd = timd.replace(hour = int(tim[0:2]), minute = int(tim[2:4]), second = int(tim[4:6]), microsecond=0) - if datetime.utcnow() < timd: # Packet was sampled yesterday + now = datetime.now(timezone.utc) + if now < timd: # Packet was sampled yesterday timd -= timedelta(1) + # Decode GPS Fix Type + new = ((ord(posi[12])-33) >> 5) & 0x1 + # Decode telemetry teld = [0]*6 if tel != None: @@ -62,22 +66,22 @@ def decode_position(tim, posi, tel): t0 = ord(tel[i*2+1]) - 33 teld.append(t0 + t1*91) - return timd,x,y,z,teld + return timd,x,y,z,teld,new def insert_position(sqlite, call, tim, posi, comm, tel): #sqlite, call, data # Decode - timd,x,y,z,teld = decode_position(tim, posi, tel) + timd,x,y,z,teld,new = decode_position(tim, posi, tel) # Insert sqlite.cursor().execute(""" - INSERT OR REPLACE INTO position (call,time,org,lat,lon,alt,comment,sequ,tel1,tel2,tel3,tel4,tel5) - VALUES (?,?,'pos',?,?,?,?,?,?,?,?,?,?)""", - (call, int(timd.timestamp()), y, x, int(z), comm, teld[0], teld[1], teld[2], teld[3], teld[4], teld[5]) + INSERT OR REPLACE INTO position (call,time,org,lat,lon,alt,new,comment,sequ,tel1,tel2,tel3,tel4,tel5) + VALUES (?,?,'pos',?,?,?,?,?,?,?,?,?,?,?)""", + (call, int(timd.timestamp()), y, x, int(z), new, comm, teld[0], teld[1], teld[2], teld[3], teld[4], teld[5]) ) sqlite.commit() # Debug tim_stringified = timd.strftime("%Y-%m-%d %H:%M:%S") - print("Decoded position from %s time %s => lat=%f lon=%f alt=%d comment=%s, sequ=%d tel=[%d,%d,%d,%d,%d]" - % (call, tim_stringified, y, x, z, comm, teld[0], teld[1], teld[2], teld[3], teld[4], teld[5])) + print("Decoded position from %s time %s => lat=%f lon=%f alt=%d new=%d comment=%s, sequ=%d tel=[%d,%d,%d,%d,%d]" + % (call, tim_stringified, y, x, int(z), new, comm, teld[0], teld[1], teld[2], teld[3], teld[4], teld[5])) diff --git a/tracker/software/config.c b/tracker/software/config.c index e804269..5454b9a 100644 --- a/tracker/software/config.c +++ b/tracker/software/config.c @@ -364,8 +364,8 @@ module_conf_t config[7]; uint8_t ssdv_buffer[128*1024] __attribute__((aligned(32))); // Image buffer -systime_t track_cycle_time = S2ST(60); // Tracking cycle (all peripheral data [airpressure, GPS, temperature, ...] is collected each 60 seconds -systime_t log_cycle_time = S2ST(60); // Log cycle time in seconds (600 seconds) +systime_t track_cycle_time = S2ST(30); // Tracking cycle (all peripheral data [airpressure, GPS, temperature, ...] is collected each 60 seconds +systime_t log_cycle_time = S2ST(30); // Log cycle time in seconds (600 seconds) bool keep_cam_switched_on = false; // Keep camera switched on and initialized, this makes image capturing faster but takes a lot of power over long time uint16_t gps_on_vbat = 3000; // Battery voltage threshold at which GPS is switched on uint16_t gps_off_vbat = 2500; // Battery voltage threshold at which GPS is switched off diff --git a/tracker/software/threads/image.c b/tracker/software/threads/image.c index 687f55b..0685538 100644 --- a/tracker/software/threads/image.c +++ b/tracker/software/threads/image.c @@ -283,7 +283,8 @@ void encode_ssdv(const uint8_t *image, uint32_t image_len, module_conf_t* conf, { ssdv_t ssdv; uint8_t pkt[SSDV_PKT_SIZE]; - uint8_t pkt_base91[270]; + uint8_t pkt_base91i[150]; + uint8_t pkt_base91j[150]; const uint8_t *b; uint32_t bi = 0; uint8_t c = SSDV_OK; @@ -361,11 +362,24 @@ void encode_ssdv(const uint8_t *image, uint32_t image_len, module_conf_t* conf, case PROT_APRS_AFSK: // Encode packet TRACE_INFO("IMG > Encode APRS/SSDV packet"); - base91_encode(&pkt[6], pkt_base91, sizeof(pkt)-42); // Sync byte, CRC and FEC of SSDV not transmitted (because its not neccessary inside an APRS packet) - aprs_encode_data_packet(&ax25_handle, 'I', &conf->aprs_conf, pkt_base91, strlen((char*)pkt_base91), captureLocation); + // Sync byte, CRC and FEC of SSDV not transmitted (because its not neccessary inside an APRS packet) + pkt[3] = pkt[6]; + pkt[4] = pkt[7]; + pkt[5] = pkt[8]; + base91_encode(&pkt[3 ], pkt_base91i, 110); + pkt[110] = pkt[6]; + pkt[111] = pkt[7]; + pkt[112] = pkt[8]; + base91_encode(&pkt[3+107], pkt_base91j, 110); + + aprs_encode_data_packet(&ax25_handle, 'I', &conf->aprs_conf, pkt_base91i, strlen((char*)pkt_base91i), captureLocation); + aprs_encode_data_packet(&ax25_handle, 'J', &conf->aprs_conf, pkt_base91j, strlen((char*)pkt_base91j), captureLocation); if(redudantTx) - aprs_encode_data_packet(&ax25_handle, 'I', &conf->aprs_conf, pkt_base91, strlen((char*)pkt_base91), captureLocation); + { + aprs_encode_data_packet(&ax25_handle, 'I', &conf->aprs_conf, pkt_base91i, strlen((char*)pkt_base91i), captureLocation); + aprs_encode_data_packet(&ax25_handle, 'J', &conf->aprs_conf, pkt_base91j, strlen((char*)pkt_base91j), captureLocation); + } // Transmit if(ax25_handle.size >= 58000 || conf->packet_spacing) // Transmit if buffer is almost full or if single packet transmission is activated (packet_spacing != 0)