diff --git a/README.md b/README.md index 523aa0c..ebc0600 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ It requires [PostgreSQL](http://www.postgresql.org/), [PostGIS](http://www.postg 4. Install [PostgreSQL](http://www.postgresql.org/) with [PostGIS](http://www.postgis.net/) and [TimescaleDB](https://www.timescale.com) Extension. Create a database (use "ogn" as default, otherwise you have to modify the configuration, see below) -5. Optional: Install redis for asynchronous tasks (like takeoff/landing-detection) +5. Install redis for asynchronous tasks (like database feeding, takeoff/landing-detection, ...) ``` apt-get install redis-server @@ -53,7 +53,7 @@ It requires [PostgreSQL](http://www.postgresql.org/), [PostGIS](http://www.postg ./flask database init ``` -8. Optional: Prepare tables for TimescaleDB +8. Prepare tables for TimescaleDB ``` ./flask database init_timescaledb @@ -72,37 +72,27 @@ It requires [PostgreSQL](http://www.postgresql.org/), [PostGIS](http://www.postg 10. Get world elevation data (needed for AGL calculation) Sources: There are many sources for DEM data. It is important that the spatial reference system (SRID) is the same as the database which is 4326. The [GMTED2010 Viewer](https://topotools.cr.usgs.gov/gmted_viewer/viewer.htm) provides data for the world with SRID 4326. Just download the data you need. - - For Europe we can get the DEM as GeoTIFF files from the [European Environment Agency](https://land.copernicus.eu/imagery-in-situ/eu-dem/eu-dem-v1.1). - Because the SRID of these files is 3035 and we want 4326 we have to convert them (next step) -11. Optional: Convert the elevation data into correct SRID - - We convert elevation from one SRID (here: 3035) to target SRID (4326): + +11. Import the GeoTIFF into the elevation table: ``` - gdalwarp -s_srs "EPSG:3035" -t_srs "EPSG:4326" source.tif target.tif - ``` - -12. Import the GeoTIFF into the elevation table: - - ``` - raster2pgsql -s 4326 -c -C -I -M -t 100x100 elevation_data.tif public.elevation | psql -d ogn + raster2pgsql *.tif -s 4326 -d -M -C -I -F -t 25x25 public.elevation | psql -d ogn ``` -13. Import Airports (needed for takeoff and landing calculation). A cup file is provided under tests: +12. Import Airports (needed for takeoff and landing calculation). A cup file is provided under tests: ``` flask database import_airports tests/SeeYou.cup ``` -14. Import DDB (needed for registration signs in the logbook). +13. Import DDB (needed for registration signs in the logbook). ``` flask database import_ddb ``` -15. Optional: Use supervisord +14. Optional: Use supervisord You can use [Supervisor](http://supervisord.org/) to control the complete system. In the directory deployment/supervisor we have some configuration files to feed the database (ogn-feed), run the celery worker (celeryd), the celery beat (celerybeatd), the celery monitor (flower), and the python wsgi server (gunicorn). All files assume that diff --git a/app/__init__.py b/app/__init__.py index facab40..59e43c2 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,3 +1,5 @@ +import os + from flask import Flask from flask_bootstrap import Bootstrap from flask_sqlalchemy import SQLAlchemy @@ -33,14 +35,17 @@ def create_app(config_name='default'): redis_client.init_app(app) init_celery(app) + register_blueprints(app) + return app + +def register_blueprints(app): from app.main import bp as bp_main app.register_blueprint(bp_main) - return app - -def init_celery(app): - celery.conf.broker_url = app.config['CELERY_BROKER_URL'] +def init_celery(app=None): + app = app or create_app(os.getenv('FLASK_CONFIG') or 'default') + celery.conf.broker_url = app.config['BROKER_URL'] celery.conf.result_backend = app.config['CELERY_RESULT_BACKEND'] celery.conf.update(app.config) @@ -52,8 +57,3 @@ def init_celery(app): celery.Task = ContextTask return celery - -# Do we need this? Otherwise I cant the celery worker run... -app = create_app() -from app.gateway.bulkimport import DbFeeder -from app.collect.celery_tasks import * \ No newline at end of file diff --git a/app/collect/celery_tasks.py b/app/collect/celery_tasks.py deleted file mode 100644 index 86b539d..0000000 --- a/app/collect/celery_tasks.py +++ /dev/null @@ -1,111 +0,0 @@ -import datetime - -from celery.utils.log import get_task_logger - -from app.collect.takeoff_landings import update_entries as takeoff_update_entries - -from app.collect.logbook import update_entries as logbook_update_entries -from app.collect.logbook import update_max_altitudes as logbook_update_max_altitudes - -from app.collect.database import import_ddb as device_infos_import_ddb -from app.collect.database import update_country_code as receivers_update_country_code - -from app.collect.ognrange import update_entries as receiver_coverage_update_entries - -from app.gateway.bulkimport import DbFeeder - -from app import db -from app import redis_client, celery - -logger = get_task_logger(__name__) - -@celery.task(name="update_takeoff_landings") -def update_takeoff_landings(last_minutes): - """Compute takeoffs and landings.""" - - end = datetime.datetime.utcnow() - start = end - datetime.timedelta(minutes=last_minutes) - result = takeoff_update_entries(session=db.session, start=start, end=end, logger=logger) - return result - - -@celery.task(name="update_logbook_entries") -def update_logbook_entries(day_offset): - """Add/update logbook entries.""" - - date = datetime.datetime.today() + datetime.timedelta(days=day_offset) - result = logbook_update_entries(session=db.session, date=date, logger=logger) - return result - - -@celery.task(name="update_logbook_max_altitude") -def update_logbook_max_altitude(day_offset): - """Add max altitudes in logbook when flight is complete (takeoff and landing).""" - - date = datetime.datetime.today() + datetime.timedelta(days=day_offset) - result = logbook_update_max_altitudes(session=db.session, date=date, logger=logger) - return result - - -@celery.task(name="import_ddb") -def import_ddb(): - """Import registered devices from the DDB.""" - - result = device_infos_import_ddb(session=db.session, logger=logger) - return result - - -@celery.task(name="update_receivers_country_code") -def update_receivers_country_code(): - """Update country code in receivers table if None.""" - - result = receivers_update_country_code(session=db.session, logger=logger) - return result - - -@celery.task(name="purge_old_data") -def purge_old_data(max_hours): - """Delete AircraftBeacons and ReceiverBeacons older than given 'age'.""" - - from app.model import AircraftBeacon, ReceiverBeacon - - min_timestamp = datetime.datetime.utcnow() - datetime.timedelta(hours=max_hours) - aircraft_beacons_deleted = db.session.query(AircraftBeacon).filter(AircraftBeacon.timestamp < min_timestamp).delete() - - receiver_beacons_deleted = db.session.query(ReceiverBeacon).filter(ReceiverBeacon.timestamp < min_timestamp).delete() - - db.session.commit() - - result = "{} AircraftBeacons deleted, {} ReceiverBeacons deleted".format(aircraft_beacons_deleted, receiver_beacons_deleted) - return result - - -@celery.task(name="update_ognrange") -def update_ognrange(day_offset): - """Create receiver coverage stats for Melissas ognrange.""" - - date = datetime.datetime.today() + datetime.timedelta(days=day_offset) - - receiver_coverage_update_entries(session=db.session, date=date) - - -@celery.task(name="transfer_beacons_to_database") -def transfer_beacons_to_database(): - """Transfer beacons from redis to TimescaleDB.""" - - counter = 0 - with DbFeeder() as feeder: - for key in redis_client.scan_iter(match="ogn-python *"): - value = redis_client.get(key) - if value is None: - redis_client.delete(key) - continue - - reference_timestamp = datetime.datetime.strptime(key[11:].decode('utf-8'), "%Y-%m-%d %H:%M:%S.%f") - aprs_string = value.decode('utf-8') - redis_client.delete(key) - - feeder.add(aprs_string, reference_timestamp=reference_timestamp) - counter += 1 - - return f"Beacons transfered from redis to TimescaleDB: {counter}" diff --git a/app/collect/database.py b/app/collect/database.py index 6193a92..acd3fcc 100644 --- a/app/collect/database.py +++ b/app/collect/database.py @@ -2,11 +2,12 @@ from sqlalchemy.sql import null, and_, func, case from sqlalchemy.dialects.postgresql import insert from flask import current_app -from app.model import Country, DeviceInfo, DeviceInfoOrigin, Receiver +from app import db +from app.model import SenderInfo, SenderInfoOrigin, Receiver from app.utils import get_ddb, get_flarmnet -def upsert(session, model, rows, update_cols): +def upsert(model, rows, update_cols): """Insert rows in model. On conflicting update columns if new value IS NOT NULL.""" table = model.__table__ @@ -18,55 +19,36 @@ def upsert(session, model, rows, update_cols): ) # print(compile_query(on_conflict_stmt)) - session.execute(on_conflict_stmt) + return on_conflict_stmt -def update_device_infos(session, address_origin, path=None): - if address_origin == DeviceInfoOrigin.FLARMNET: +def update_device_infos(address_origin, path=None): + if address_origin == SenderInfoOrigin.FLARMNET: device_infos = get_flarmnet(fln_file=path) else: device_infos = get_ddb(csv_file=path) - session.query(DeviceInfo).filter(DeviceInfo.address_origin == address_origin).delete(synchronize_session="fetch") - session.commit() + db.session.query(SenderInfo).filter(SenderInfo.address_origin == address_origin).delete(synchronize_session="fetch") + db.session.commit() for device_info in device_infos: device_info.address_origin = address_origin - session.bulk_save_objects(device_infos) - session.commit() + db.session.bulk_save_objects(device_infos) + db.session.commit() return len(device_infos) -def import_ddb(session, logger=None): +def import_ddb(logger=None): """Import registered devices from the DDB.""" if logger is None: logger = current_app.logger logger.info("Import registered devices fom the DDB...") - counter = update_device_infos(session, DeviceInfoOrigin.OGN_DDB) + counter = update_device_infos(SenderInfoOrigin.OGN_DDB) - finish_message = "DeviceInfo: {} inserted.".format(counter) - logger.info(finish_message) - return finish_message - - -def update_country_code(session, logger=None): - """Update country code in receivers table if None.""" - - if logger is None: - logger = current_app.logger - - update_receivers = ( - session.query(Receiver) - .filter(and_(Receiver.country_id == null(), Receiver.location_wkt != null(), func.st_within(Receiver.location_wkt, Country.geom))) - .update({Receiver.country_id: Country.gid}, synchronize_session="fetch") - ) - - session.commit() - - finish_message = "Receivers (country): {} updated".format(update_receivers) + finish_message = "SenderInfo: {} inserted.".format(counter) logger.info(finish_message) return finish_message diff --git a/app/collect/flights.py b/app/collect/flights.py new file mode 100644 index 0000000..922ee4a --- /dev/null +++ b/app/collect/flights.py @@ -0,0 +1,113 @@ +from datetime import date + +from app import db + +NOTHING = "" +CONTEST_RELEVANT = "AND agl < 1000" +LOW_PASS = "AND agl < 50 and ground_speed > 250" + +def compute_flights(date, flight_type=0): + if flight_type == 0: + filter = NOTHING + elif flight_type == 1: + filter = CONTEST_RELEVANT + elif flight_type == 2: + filter = LOW_PASS + + date_str = date.strftime("%Y-%m-%d") + + query = f""" + INSERT INTO flights(date, sender_id, flight_type, multilinestring, simple_multilinestring) + SELECT '{date_str}' AS date, + s.id AS sender_id, + {flight_type} as flight_type, + st_collect(sq5.linestring order BY sq5.part) multilinestring, + st_collect(st_simplify(sq5.linestring, 0.0001) ORDER BY sq5.part) simple_multilinestring + FROM ( + SELECT sq4.name, + sq4.part, + st_makeline(sq4.location ORDER BY sq4.timestamp) AS linestring + FROM ( + SELECT sq3.timestamp, + sq3.location, + sq3.name, + SUM(sq3.ping) OVER (partition BY sq3.name ORDER BY sq3.timestamp) AS part + FROM ( + SELECT sq2.t1 AS timestamp, + sq2.l1 AS location, + sq2.s1 AS name, + CASE + WHEN sq2.s1 = sq2.s2 AND sq2.t1 - sq2.t2 < interval'100s' AND ST_DistanceSphere(sq2.l1, sq2.l2) < 1000 THEN 0 + ELSE 1 + END AS ping + FROM ( + SELECT sq.timestamp t1, + lag(sq.timestamp) OVER (partition BY sq.name ORDER BY sq.timestamp) t2, + sq.location l1, + lag(sq.location) OVER (partition BY sq.name ORDER BY sq.timestamp) l2, + sq.name s1, + lag(sq.name) OVER (partition BY sq.name ORDER BY sq.timestamp) s2 + FROM ( + SELECT DISTINCT ON (name, timestamp) name, timestamp, location + FROM sender_positions + WHERE reference_timestamp BETWEEN '{date_str} 00:00:00' AND '{date_str} 23:59:59' {filter} + ORDER BY name, timestamp, error_count + ) AS sq + ) AS sq2 + ) AS sq3 + ) AS sq4 + GROUP BY sq4.name, sq4.part + ) AS sq5 + INNER JOIN senders AS s ON sq5.name = s.name + GROUP BY s.id + ON CONFLICT DO NOTHING; + """ + + db.session.execute(query) + db.session.commit() + +def compute_gaps(date): + date_str = date.strftime("%Y-%m-%d") + + query = f""" + INSERT INTO flights(date, flight_type, sender_id, multilinestring) + SELECT '{date_str}' AS date, + 3 AS flight_type, + s.id AS sender_id, + ST_Collect(sq3.path) + FROM ( + SELECT sq2.s1 AS name, + ST_MakeLine(sq2.l1, sq2.l2) AS path + FROM + ( + SELECT sq.timestamp t1, + LAG(sq.timestamp) OVER (PARTITION BY sq.timestamp::DATE, sq.name ORDER BY sq.timestamp) t2, + sq.location l1, + LAG(sq.location) OVER (PARTITION BY sq.timestamp::DATE, sq.name ORDER BY sq.timestamp) l2, + sq.name s1, + LAG(sq.name) OVER (PARTITION BY sq.timestamp::DATE, sq.name ORDER BY sq.timestamp) s2 + FROM + ( + SELECT DISTINCT ON (name, timestamp) name, timestamp, location, agl + FROM sender_positions + WHERE reference_timestamp BETWEEN '{date_str} 00:00:00' AND '{date_str} 23:59:59' AND agl > 300 + ORDER BY name, timestamp, error_count + ) AS sq + ) AS sq2 + WHERE EXTRACT(epoch FROM sq2.t1 - sq2.t2) > 300 + AND ST_DistanceSphere(sq2.l1, sq2.l2) / EXTRACT(epoch FROM sq2.t1 - sq2.t2) BETWEEN 15 AND 50 + ) AS sq3 + INNER JOIN senders AS s on sq3.name = s.name + GROUP BY s.id + ON CONFLICT DO NOTHING; + """ + + db.session.execute(query) + db.session.commit() + +if __name__ == '__main__': + from app import create_app + app = create_app() + with app.app_context(): + result = compute_flights(date=date(2020, 10, 28)) + print(result) \ No newline at end of file diff --git a/app/collect/gateway.py b/app/collect/gateway.py new file mode 100644 index 0000000..f1850d1 --- /dev/null +++ b/app/collect/gateway.py @@ -0,0 +1,22 @@ +from datetime import datetime +from flask import current_app + +from app import redis_client +from app.gateway.message_handling import sender_position_csv_strings_to_db, receiver_position_csv_strings_to_db, receiver_status_csv_strings_to_db + +def transfer_from_redis_to_database(): + unmapping = lambda s: s[0].decode('utf-8') + + receiver_status_data = list(map(unmapping, redis_client.zpopmin('receiver_status', 100000))) + receiver_position_data = list(map(unmapping, redis_client.zpopmin('receiver_position', 100000))) + sender_status_data = list(map(unmapping, redis_client.zpopmin('sender_status', 100000))) + sender_position_data = list(map(unmapping, redis_client.zpopmin('sender_position', 100000))) + + receiver_status_csv_strings_to_db(lines=receiver_status_data) + receiver_position_csv_strings_to_db(lines=receiver_position_data) + sender_position_csv_strings_to_db(lines=sender_position_data) + + current_app.logger.debug(f"transfer_from_redis_to_database: rx_stat: {len(receiver_status_data):6d}\trx_pos: {len(receiver_position_data):6d}\ttx_stat: {len(sender_status_data):6d}\ttx_pos: {len(sender_position_data):6d}") + + finish_message = f"Database: {len(receiver_status_data)+len(receiver_position_data)+len(sender_status_data)+len(sender_position_data)} inserted" + return finish_message diff --git a/app/collect/logbook.py b/app/collect/logbook.py index c116c0a..e7acfeb 100644 --- a/app/collect/logbook.py +++ b/app/collect/logbook.py @@ -1,87 +1,229 @@ from sqlalchemy import and_, or_, insert, update, exists, between from sqlalchemy.sql import func, null -from sqlalchemy.sql.expression import true, false +from sqlalchemy.sql.expression import case, true, false from flask import current_app -from app.model import TakeoffLanding, Logbook, AircraftBeacon +from app.model import Airport, SenderPosition, Sender, TakeoffLanding, Logbook from app.utils import date_to_timestamps +from datetime import datetime, timedelta -def update_entries(session, date, logger=None): - """Add/update logbook entries.""" +from app import db - if logger is None: - logger = current_app.logger - logger.info("Compute logbook.") +# takeoff / landing detection is based on 3 consecutive points +MIN_TAKEOFF_SPEED = 55 # takeoff detection: 1st point below, 2nd and 3rd above this limit +MAX_LANDING_SPEED = 40 # landing detection: 1st point above, 2nd and 3rd below this limit +MIN_TAKEOFF_CLIMB_RATE = -5 # takeoff detection: glider should not sink too much +MAX_LANDING_SINK_RATE = 5 # landing detection: glider should not climb too much +MAX_EVENT_DURATION = 100 # the points must not exceed this duration +MAX_EVENT_RADIUS = 5000 # the points must not exceed this radius around the 2nd point +MAX_EVENT_AGL = 200 # takeoff / landing must not exceed this altitude AGL - # limit time range to given date and set window partition and window order - (start, end) = date_to_timestamps(date) - pa = TakeoffLanding.address - wo = and_(TakeoffLanding.address, TakeoffLanding.airport_id, TakeoffLanding.timestamp) + +def update_takeoff_landings(start, end): + """Compute takeoffs and landings.""" + + current_app.logger.info("Compute takeoffs and landings.") + + # considered time interval should not exceed a complete day + if end - start > timedelta(days=1): + abort_message = "TakeoffLanding: timeinterval start='{}' and end='{}' is too big.".format(start, end) + current_app.logger.warn(abort_message) + return abort_message + + # check if we have any airport + airports_query = db.session.query(Airport).limit(1) + if not airports_query.all(): + abort_message = "TakeoffLanding: Cannot calculate takeoff and landings without any airport! Please import airports first." + current_app.logger.warn(abort_message) + return abort_message # delete existing elements - session.query(Logbook)\ - .filter(between(Logbook.reftime, start, end))\ + db.session.query(TakeoffLanding) \ + .filter(between(TakeoffLanding.timestamp, start, end))\ .delete(synchronize_session='fetch') - session.commit() + db.session.commit() - # make a query with current, previous and next "takeoff_landing" event, so we can find complete flights + # get beacons for selected time range (+ buffer for duration), one per name and timestamp sq = ( - session.query( - TakeoffLanding.address, - func.lag(TakeoffLanding.address).over(partition_by=pa, order_by=wo).label("address_prev"), - func.lead(TakeoffLanding.address).over(partition_by=pa, order_by=wo).label("address_next"), + db.session.query(SenderPosition.name, SenderPosition.timestamp, SenderPosition.location, SenderPosition.track, db.func.coalesce(SenderPosition.ground_speed, 0.0).label("ground_speed"), SenderPosition.altitude, db.func.coalesce(SenderPosition.climb_rate, 0.0).label("climb_rate")) + .distinct(SenderPosition.name, SenderPosition.timestamp) + .order_by(SenderPosition.name, SenderPosition.timestamp, SenderPosition.error_count) + .filter(SenderPosition.agl <= MAX_EVENT_AGL) + .filter(between(SenderPosition.reference_timestamp, start - timedelta(seconds=MAX_EVENT_DURATION), end + timedelta(seconds=MAX_EVENT_DURATION))) + .subquery() + ) + + # make a query with current, previous and next position + sq2 = db.session.query( + sq.c.name, + func.lag(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_prev"), + func.lead(sq.c.name).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("name_next"), + sq.c.timestamp, + func.lag(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_prev"), + func.lead(sq.c.timestamp).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("timestamp_next"), + sq.c.location, + func.lag(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_prev"), + func.lead(sq.c.location).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("location_wkt_next"), + sq.c.track, + func.lag(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_prev"), + func.lead(sq.c.track).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("track_next"), + sq.c.ground_speed, + func.lag(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_prev"), + func.lead(sq.c.ground_speed).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("ground_speed_next"), + sq.c.altitude, + func.lag(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_prev"), + func.lead(sq.c.altitude).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("altitude_next"), + sq.c.climb_rate, + func.lag(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_prev"), + func.lead(sq.c.climb_rate).over(partition_by=sq.c.name, order_by=sq.c.timestamp).label("climb_rate_next"), + ).subquery() + + # consider only positions between start and end and with predecessor and successor and limit distance and duration between points + sq3 = ( + db.session.query(sq2) + .filter(and_(sq2.c.name_prev != null(), sq2.c.name_next != null())) + .filter(and_(func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < MAX_EVENT_RADIUS, func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < MAX_EVENT_RADIUS)) + .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=MAX_EVENT_DURATION)) + .filter(between(sq2.c.timestamp, start, end)) + .subquery() + ) + + # find possible takeoffs and landings + sq4 = ( + db.session.query( + sq3.c.timestamp, + case( + [ + (sq3.c.ground_speed > MIN_TAKEOFF_SPEED, sq3.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport + (sq3.c.ground_speed <= MIN_TAKEOFF_SPEED, sq3.c.location), + ] + ).label("location"), + case([(sq3.c.ground_speed > MAX_LANDING_SPEED, sq3.c.track), (sq3.c.ground_speed <= MAX_LANDING_SPEED, sq3.c.track_prev)]).label( + "track" + ), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly + sq3.c.ground_speed, + sq3.c.altitude, + case([(sq3.c.ground_speed > MIN_TAKEOFF_SPEED, True), (sq3.c.ground_speed < MAX_LANDING_SPEED, False)]).label("is_takeoff"), + sq3.c.name, + ) + .filter( + or_( + and_(sq3.c.ground_speed_prev < MIN_TAKEOFF_SPEED, sq3.c.ground_speed > MIN_TAKEOFF_SPEED, sq3.c.ground_speed_next > MIN_TAKEOFF_SPEED, sq3.c.climb_rate > MIN_TAKEOFF_CLIMB_RATE), # takeoff + and_(sq3.c.ground_speed_prev > MAX_LANDING_SPEED, sq3.c.ground_speed < MAX_LANDING_SPEED, sq3.c.ground_speed_next < MAX_LANDING_SPEED, sq3.c.climb_rate < MAX_LANDING_SINK_RATE), # landing + ) + ) + .subquery() + ) + + # get the device id instead of the name and consider them if the are near airports ... + sq5 = ( + db.session.query( + sq4.c.timestamp, sq4.c.track, sq4.c.is_takeoff, Sender.id.label("device_id"), Airport.id.label("airport_id"), func.ST_DistanceSphere(sq4.c.location, Airport.location_wkt).label("airport_distance") + ) + .filter(and_(func.ST_Within(sq4.c.location, Airport.border), + between(Airport.style, 2, 5))) + .filter(sq4.c.name == Sender.name) + .subquery() + ) + + # ... and take the nearest airport + takeoff_landing_query = ( + db.session.query(sq5.c.timestamp, sq5.c.track, sq5.c.is_takeoff, sq5.c.device_id, sq5.c.airport_id) + .distinct(sq5.c.timestamp, sq5.c.track, sq5.c.is_takeoff, sq5.c.device_id) + .order_by(sq5.c.timestamp, sq5.c.track, sq5.c.is_takeoff, sq5.c.device_id, sq5.c.airport_distance) + .subquery() + ) + + # ... and save them + ins = insert(TakeoffLanding).from_select((TakeoffLanding.timestamp, TakeoffLanding.track, TakeoffLanding.is_takeoff, TakeoffLanding.sender_id, TakeoffLanding.airport_id), takeoff_landing_query) + + result = db.session.execute(ins) + db.session.commit() + insert_counter = result.rowcount + + finish_message = "TakeoffLandings: {} inserted".format(insert_counter) + current_app.logger.info(finish_message) + return finish_message + + +if __name__ == '__main__': + from app import create_app + app = create_app() + with app.app_context(): + result = update_takeoff_landings(start=datetime(2020, 11, 9, 10, 0, 0), end=datetime(2020, 11, 9, 10, 10, 0)) + print(result) + + +def update_logbook(offset_days): + """Add/update logbook entries.""" + + current_app.logger.info("Compute logbook.") + + # limit time range to given date and set window partition and window order + (start, end) = date_to_timestamps(datetime.utcnow()-timedelta(days=offset_days)) + pa = TakeoffLanding.sender_id + wo = and_(TakeoffLanding.sender_id, TakeoffLanding.timestamp, TakeoffLanding.airport_id) + + # delete existing elements + db.session.query(Logbook)\ + .filter(between(Logbook.reference, start, end))\ + .delete(synchronize_session='fetch') + db.session.commit() + + # make a query with current and next "takeoff_landing" event, so we can find complete flights + sq = ( + db.session.query( + TakeoffLanding.sender_id, + func.lead(TakeoffLanding.sender_id).over(partition_by=pa, order_by=wo).label("sender_id_next"), TakeoffLanding.timestamp, - func.lag(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_prev"), func.lead(TakeoffLanding.timestamp).over(partition_by=pa, order_by=wo).label("timestamp_next"), TakeoffLanding.track, - func.lag(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_prev"), func.lead(TakeoffLanding.track).over(partition_by=pa, order_by=wo).label("track_next"), TakeoffLanding.is_takeoff, - func.lag(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_prev"), func.lead(TakeoffLanding.is_takeoff).over(partition_by=pa, order_by=wo).label("is_takeoff_next"), TakeoffLanding.airport_id, - func.lag(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_prev"), - func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next"), + func.lead(TakeoffLanding.airport_id).over(partition_by=pa, order_by=wo).label("airport_id_next") ) .filter(between(TakeoffLanding.timestamp, start, end)) .subquery() ) # find complete flights - complete_flight_query = session.query( - sq.c.timestamp.label("reftime"), - sq.c.address.label("address"), - sq.c.timestamp.label("takeoff_timestamp"), - sq.c.track.label("takeoff_track"), - sq.c.airport_id.label("takeoff_airport_id"), - sq.c.timestamp_next.label("landing_timestamp"), - sq.c.track_next.label("landing_track"), - sq.c.airport_id_next.label("landing_airport_id"), - ).filter(and_(sq.c.is_takeoff == true(), sq.c.is_takeoff_next == false())) + complete_flight_query = ( + db.session.query( + sq.c.sender_id.label("sender_id"), + sq.c.timestamp.label("takeoff_timestamp"), + sq.c.track.label("takeoff_track"), + sq.c.airport_id.label("takeoff_airport_id"), + sq.c.timestamp_next.label("landing_timestamp"), + sq.c.track_next.label("landing_track"), + sq.c.airport_id_next.label("landing_airport_id"), + ) + .filter(sq.c.is_takeoff == true()) + .filter(sq.c.is_takeoff_next == false()) + ) # find landings without start only_landings_query = ( - session.query( - sq.c.timestamp.label("reftime"), - sq.c.address.label("address"), + db.session.query( + sq.c.sender_id_next.label("sender_id"), null().label("takeoff_timestamp"), null().label("takeoff_track"), null().label("takeoff_airport_id"), - sq.c.timestamp.label("landing_timestamp"), - sq.c.track.label("landing_track"), - sq.c.airport_id.label("landing_airport_id"), + sq.c.timestamp_next.label("landing_timestamp"), + sq.c.track_next.label("landing_track"), + sq.c.airport_id_next.label("landing_airport_id"), ) - .filter(sq.c.is_takeoff == false()) - .filter(or_(sq.c.is_takeoff_prev == false(), sq.c.is_takeoff_prev == null())) + .filter(or_(sq.c.is_takeoff == false(), sq.c.is_takeoff == null())) + .filter(sq.c.is_takeoff_next == false()) ) # find starts without landing only_starts_query = ( - session.query( - sq.c.timestamp.label("reftime"), - sq.c.address.label("address"), + db.session.query( + sq.c.sender_id.label("sender_id"), sq.c.timestamp.label("takeoff_timestamp"), sq.c.track.label("takeoff_track"), sq.c.airport_id.label("takeoff_airport_id"), @@ -99,8 +241,7 @@ def update_entries(session, date, logger=None): # ... insert them into logbook ins = insert(Logbook).from_select( ( - Logbook.reftime, - Logbook.address, + Logbook.sender_id, Logbook.takeoff_timestamp, Logbook.takeoff_track, Logbook.takeoff_airport_id, @@ -111,47 +252,42 @@ def update_entries(session, date, logger=None): logbook_entries, ) - result = session.execute(ins) + result = db.session.execute(ins) insert_counter = result.rowcount - session.commit() + db.session.commit() finish_message = "Logbook: {} inserted".format(insert_counter) - logger.debug(finish_message) return finish_message -def update_max_altitudes(session, date, logger=None): +def update_max_altitudes(date, logger=None): """Add max altitudes in logbook when flight is complete (takeoff and landing).""" if logger is None: logger = current_app.logger - logger.info("Update logbook max altitude.") - - if session is None: - session = current_app.session + current_app.logger.info("Update logbook max altitude.") (start, end) = date_to_timestamps(date) logbook_entries = ( - session.query(Logbook.id) + db.query(Logbook.id) .filter(and_(Logbook.takeoff_timestamp != null(), Logbook.landing_timestamp != null(), Logbook.max_altitude == null())) - .filter(between(Logbook.reftime, start, end)) + .filter(between(Logbook.reference, start, end)) .subquery() ) max_altitudes = ( - session.query(Logbook.id, func.max(AircraftBeacon.altitude).label("max_altitude")) + db.query(Logbook.id, func.max(SenderPosition.altitude).label("max_altitude")) .filter(Logbook.id == logbook_entries.c.id) - .filter(and_(AircraftBeacon.address == Logbook.address, AircraftBeacon.timestamp >= Logbook.takeoff_timestamp, AircraftBeacon.timestamp <= Logbook.landing_timestamp)) + .filter(and_(SenderPosition.address == Logbook.address, SenderPosition.timestamp >= Logbook.takeoff_timestamp, SenderPosition.timestamp <= Logbook.landing_timestamp)) .group_by(Logbook.id) .subquery() ) - update_logbook = session.query(Logbook).filter(Logbook.id == max_altitudes.c.id).update({Logbook.max_altitude: max_altitudes.c.max_altitude}, synchronize_session="fetch") + update_logbook = db.query(Logbook).filter(Logbook.id == max_altitudes.c.id).update({Logbook.max_altitude: max_altitudes.c.max_altitude}, synchronize_session="fetch") - session.commit() + db.session.commit() finish_message = "Logbook (altitude): {} entries updated.".format(update_logbook) - logger.info(finish_message) return finish_message diff --git a/app/collect/ognrange.py b/app/collect/ognrange.py deleted file mode 100644 index bca7ab4..0000000 --- a/app/collect/ognrange.py +++ /dev/null @@ -1,104 +0,0 @@ -from sqlalchemy import Date -from sqlalchemy import and_, insert, update, exists, between -from sqlalchemy.sql import func, null -from flask import current_app - -from app.model import AircraftBeacon, Receiver, ReceiverCoverage -from app.utils import date_to_timestamps - - -def update_entries(session, date, logger=None): - """Create receiver coverage stats for Melissas ognrange.""" - - if logger is None: - logger = current_app.logger - - logger.info("Compute receiver coverages.") - - (start, end) = date_to_timestamps(date) - - # Filter aircraft beacons - sq = ( - session.query(AircraftBeacon.location_mgrs_short, AircraftBeacon.receiver_name, AircraftBeacon.signal_quality, AircraftBeacon.altitude, AircraftBeacon.address) - .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.location_mgrs_short != null(), AircraftBeacon.receiver_name != null(), AircraftBeacon.address != null())) - .subquery() - ) - - # ... and group them by reduced MGRS, receiver and date - sq2 = ( - session.query( - sq.c.location_mgrs_short, - sq.c.receiver_name, - func.cast(date, Date).label("date"), - func.max(sq.c.signal_quality).label("max_signal_quality"), - func.min(sq.c.altitude).label("min_altitude"), - func.max(sq.c.altitude).label("max_altitude"), - func.count(sq.c.altitude).label("aircraft_beacon_count"), - func.count(func.distinct(sq.c.address)).label("device_count"), - ) - .group_by(sq.c.location_mgrs_short, sq.c.receiver_name) - .subquery() - ) - - # Replace receiver_name with receiver_id - sq3 = ( - session.query( - sq2.c.location_mgrs_short, - Receiver.id.label("receiver_id"), - sq2.c.date, - sq2.c.max_signal_quality, - sq2.c.min_altitude, - sq2.c.max_altitude, - sq2.c.aircraft_beacon_count, - sq2.c.device_count, - ) - .filter(sq2.c.receiver_name == Receiver.name) - .subquery() - ) - - # if a receiver coverage entry exist --> update it - upd = ( - update(ReceiverCoverage) - .where(and_(ReceiverCoverage.location_mgrs_short == sq3.c.location_mgrs_short, ReceiverCoverage.receiver_id == sq3.c.receiver_id, ReceiverCoverage.date == date)) - .values( - { - "max_signal_quality": sq3.c.max_signal_quality, - "min_altitude": sq3.c.min_altitude, - "max_altitude": sq3.c.max_altitude, - "aircraft_beacon_count": sq3.c.aircraft_beacon_count, - "device_count": sq3.c.device_count, - } - ) - ) - - result = session.execute(upd) - update_counter = result.rowcount - session.commit() - logger.debug("Updated receiver coverage entries: {}".format(update_counter)) - - # if a receiver coverage entry doesnt exist --> insert it - new_coverage_entries = session.query(sq3).filter( - ~exists().where(and_(ReceiverCoverage.location_mgrs_short == sq3.c.location_mgrs_short, ReceiverCoverage.receiver_id == sq3.c.receiver_id, ReceiverCoverage.date == date)) - ) - - ins = insert(ReceiverCoverage).from_select( - ( - ReceiverCoverage.location_mgrs_short, - ReceiverCoverage.receiver_id, - ReceiverCoverage.date, - ReceiverCoverage.max_signal_quality, - ReceiverCoverage.min_altitude, - ReceiverCoverage.max_altitude, - ReceiverCoverage.aircraft_beacon_count, - ReceiverCoverage.device_count, - ), - new_coverage_entries, - ) - - result = session.execute(ins) - insert_counter = result.rowcount - session.commit() - - finish_message = "ReceiverCoverage: {} inserted, {} updated".format(insert_counter, update_counter) - logger.debug(finish_message) - return finish_message diff --git a/app/collect/stats.py b/app/collect/stats.py deleted file mode 100644 index 8e1d352..0000000 --- a/app/collect/stats.py +++ /dev/null @@ -1,450 +0,0 @@ -from flask import current_app -from sqlalchemy import insert, distinct, between, literal -from sqlalchemy.sql import null, and_, func, or_, update -from sqlalchemy.sql.expression import case - -from app.model import AircraftBeacon, DeviceStats, Country, CountryStats, ReceiverStats, ReceiverBeacon, RelationStats, Receiver, Device -from app.utils import date_to_timestamps - - -# 40dB@10km is enough for 640km -MAX_PLAUSIBLE_QUALITY = 40 - - -def create_device_stats(session, date, logger=None): - """Add/update device stats.""" - - if logger is None: - logger = current_app.logger - - (start, end) = date_to_timestamps(date) - - # First kill the stats for the selected date - deleted_counter = session.query(DeviceStats).filter(DeviceStats.date == date).delete() - - # Since "distinct count" does not work in window functions we need a work-around for receiver counting - sq = ( - session.query(AircraftBeacon, func.dense_rank().over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.receiver_id).label("dr")) - .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.device_id != null())) - .filter(or_(AircraftBeacon.error_count == 0, AircraftBeacon.error_count == null())) - .subquery() - ) - - # Calculate stats, firstseen, lastseen and last values != NULL - device_stats = session.query( - distinct(sq.c.device_id).label("device_id"), - literal(date).label("date"), - func.max(sq.c.dr).over(partition_by=sq.c.device_id).label("receiver_count"), - func.max(sq.c.altitude).over(partition_by=sq.c.device_id).label("max_altitude"), - func.count(sq.c.device_id).over(partition_by=sq.c.device_id).label("aircraft_beacon_count"), - func.first_value(sq.c.name).over(partition_by=sq.c.device_id, order_by=case([(sq.c.name == null(), None)], else_=sq.c.timestamp).asc().nullslast()).label("name"), - func.first_value(sq.c.timestamp).over(partition_by=sq.c.device_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast()).label("firstseen"), - func.first_value(sq.c.timestamp).over(partition_by=sq.c.device_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("lastseen"), - func.first_value(sq.c.aircraft_type).over(partition_by=sq.c.device_id, order_by=case([(sq.c.aircraft_type == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("aircraft_type"), - func.first_value(sq.c.stealth).over(partition_by=sq.c.device_id, order_by=case([(sq.c.stealth == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("stealth"), - func.first_value(sq.c.software_version) - .over(partition_by=sq.c.device_id, order_by=case([(sq.c.software_version == null(), None)], else_=sq.c.timestamp).desc().nullslast()) - .label("software_version"), - func.first_value(sq.c.hardware_version) - .over(partition_by=sq.c.device_id, order_by=case([(sq.c.hardware_version == null(), None)], else_=sq.c.timestamp).desc().nullslast()) - .label("hardware_version"), - func.first_value(sq.c.real_address).over(partition_by=sq.c.device_id, order_by=case([(sq.c.real_address == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("real_address"), - ).subquery() - - # And insert them - ins = insert(DeviceStats).from_select( - [ - DeviceStats.device_id, - DeviceStats.date, - DeviceStats.receiver_count, - DeviceStats.max_altitude, - DeviceStats.aircraft_beacon_count, - DeviceStats.name, - DeviceStats.firstseen, - DeviceStats.lastseen, - DeviceStats.aircraft_type, - DeviceStats.stealth, - DeviceStats.software_version, - DeviceStats.hardware_version, - DeviceStats.real_address, - ], - device_stats, - ) - res = session.execute(ins) - insert_counter = res.rowcount - session.commit() - logger.debug("DeviceStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)) - - return "DeviceStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter) - - -def create_receiver_stats(session, date, logger=None): - """Add/update receiver stats.""" - - if logger is None: - logger = current_app.logger - - (start, end) = date_to_timestamps(date) - - # First kill the stats for the selected date - deleted_counter = session.query(ReceiverStats).filter(ReceiverStats.date == date).delete() - - # Select one day - sq = session.query(ReceiverBeacon).filter(between(ReceiverBeacon.timestamp, start, end)).subquery() - - # Calculate stats, firstseen, lastseen and last values != NULL - receiver_stats = session.query( - distinct(sq.c.receiver_id).label("receiver_id"), - literal(date).label("date"), - func.first_value(sq.c.timestamp).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).asc().nullslast()).label("firstseen"), - func.first_value(sq.c.timestamp).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.timestamp == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("lastseen"), - func.first_value(sq.c.location).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.location == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("location_wkt"), - func.first_value(sq.c.altitude).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.altitude == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("altitude"), - func.first_value(sq.c.version).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.version == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("version"), - func.first_value(sq.c.platform).over(partition_by=sq.c.receiver_id, order_by=case([(sq.c.platform == null(), None)], else_=sq.c.timestamp).desc().nullslast()).label("platform"), - ).subquery() - - # And insert them - ins = insert(ReceiverStats).from_select( - [ - ReceiverStats.receiver_id, - ReceiverStats.date, - ReceiverStats.firstseen, - ReceiverStats.lastseen, - ReceiverStats.location_wkt, - ReceiverStats.altitude, - ReceiverStats.version, - ReceiverStats.platform, - ], - receiver_stats, - ) - res = session.execute(ins) - insert_counter = res.rowcount - session.commit() - logger.warn("ReceiverStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)) - - # Update aircraft_beacon_count, aircraft_count and max_distance - aircraft_beacon_stats = ( - session.query( - AircraftBeacon.receiver_id, - func.count(AircraftBeacon.timestamp).label("aircraft_beacon_count"), - func.count(func.distinct(AircraftBeacon.device_id)).label("aircraft_count"), - func.max(AircraftBeacon.distance).label("max_distance"), - ) - .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.error_count == 0, AircraftBeacon.quality <= MAX_PLAUSIBLE_QUALITY, AircraftBeacon.relay == null())) - .group_by(AircraftBeacon.receiver_id) - .subquery() - ) - - upd = ( - update(ReceiverStats) - .where(and_(ReceiverStats.date == date, ReceiverStats.receiver_id == aircraft_beacon_stats.c.receiver_id)) - .values( - {"aircraft_beacon_count": aircraft_beacon_stats.c.aircraft_beacon_count, "aircraft_count": aircraft_beacon_stats.c.aircraft_count, "max_distance": aircraft_beacon_stats.c.max_distance} - ) - ) - - result = session.execute(upd) - update_counter = result.rowcount - session.commit() - logger.warn("Updated {} ReceiverStats".format(update_counter)) - - return "ReceiverStats for {}: {} deleted, {} inserted, {} updated".format(date, deleted_counter, insert_counter, update_counter) - - -def create_country_stats(session, date, logger=None): - if logger is None: - logger = current_app.logger - - (start, end) = date_to_timestamps(date) - - # First kill the stats for the selected date - deleted_counter = session.query(CountryStats).filter(CountryStats.date == date).delete() - - country_stats = ( - session.query(literal(date), Country.gid, func.count(AircraftBeacon.timestamp).label("aircraft_beacon_count"), func.count(func.distinct(AircraftBeacon.receiver_id)).label("device_count")) - .filter(between(AircraftBeacon.timestamp, start, end)) - .filter(func.st_contains(Country.geom, AircraftBeacon.location)) - .group_by(Country.gid) - .subquery() - ) - - # And insert them - ins = insert(CountryStats).from_select([CountryStats.date, CountryStats.country_id, CountryStats.aircraft_beacon_count, CountryStats.device_count], country_stats) - res = session.execute(ins) - insert_counter = res.rowcount - session.commit() - - -def update_device_stats_jumps(session, date, logger=None): - """Update device stats jumps.""" - - if logger is None: - logger = current_app.logger - - (start, end) = date_to_timestamps(date) - - # speed limits in m/s (values above indicates a unplausible position / jump) - max_horizontal_speed = 1000 - max_vertical_speed = 100 - max_jumps = 10 # threshold for an 'ambiguous' device - - # find consecutive positions for a device - sq = ( - session.query( - AircraftBeacon.device_id, - AircraftBeacon.timestamp, - func.lead(AircraftBeacon.timestamp).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label("timestamp_next"), - AircraftBeacon.location_wkt, - func.lead(AircraftBeacon.location_wkt).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label("location_next"), - AircraftBeacon.altitude, - func.lead(AircraftBeacon.altitude).over(partition_by=AircraftBeacon.device_id, order_by=AircraftBeacon.timestamp).label("altitude_next"), - ) - .filter(and_(between(AircraftBeacon.timestamp, start, end), AircraftBeacon.error_count == 0)) - .subquery() - ) - - # calc vertial and horizontal speed between points - sq2 = ( - session.query( - sq.c.device_id, - (func.st_distancesphere(sq.c.location_next, sq.c.location) / (func.extract("epoch", sq.c.timestamp_next) - func.extract("epoch", sq.c.timestamp))).label("horizontal_speed"), - ((sq.c.altitude_next - sq.c.altitude) / (func.extract("epoch", sq.c.timestamp_next) - func.extract("epoch", sq.c.timestamp))).label("vertical_speed"), - ) - .filter(and_(sq.c.timestamp != null(), sq.c.timestamp_next != null(), sq.c.timestamp < sq.c.timestamp_next)) - .subquery() - ) - - # ... and find and count 'jumps' - sq3 = ( - session.query(sq2.c.device_id, func.sum(case([(or_(func.abs(sq2.c.horizontal_speed) > max_horizontal_speed, func.abs(sq2.c.vertical_speed) > max_vertical_speed), 1)], else_=0)).label("jumps")) - .group_by(sq2.c.device_id) - .subquery() - ) - - upd = update(DeviceStats).where(and_(DeviceStats.date == date, DeviceStats.device_id == sq3.c.device_id)).values({"ambiguous": sq3.c.jumps > max_jumps, "jumps": sq3.c.jumps}) - - result = session.execute(upd) - update_counter = result.rowcount - session.commit() - logger.warn("Updated {} DeviceStats jumps".format(update_counter)) - - return "DeviceStats jumps for {}: {} updated".format(date, update_counter) - - -def create_relation_stats(session, date, logger=None): - """Add/update relation stats.""" - - if logger is None: - logger = current_app.logger - - (start, end) = date_to_timestamps(date) - - # First kill the stats for the selected date - deleted_counter = session.query(RelationStats).filter(RelationStats.date == date).delete() - - # Calculate stats for selected day - relation_stats = ( - session.query(literal(date), AircraftBeacon.device_id, AircraftBeacon.receiver_id, func.max(AircraftBeacon.quality), func.count(AircraftBeacon.timestamp)) - .filter( - and_( - between(AircraftBeacon.timestamp, start, end), - AircraftBeacon.distance > 1000, - AircraftBeacon.error_count == 0, - AircraftBeacon.quality <= MAX_PLAUSIBLE_QUALITY, - AircraftBeacon.ground_speed > 10, - ) - ) - .group_by(literal(date), AircraftBeacon.device_id, AircraftBeacon.receiver_id) - .subquery() - ) - - # And insert them - ins = insert(RelationStats).from_select([RelationStats.date, RelationStats.device_id, RelationStats.receiver_id, RelationStats.quality, RelationStats.beacon_count], relation_stats) - res = session.execute(ins) - insert_counter = res.rowcount - session.commit() - logger.warn("RelationStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter)) - - return "RelationStats for {}: {} deleted, {} inserted".format(date, deleted_counter, insert_counter) - - -def update_qualities(session, date, logger=None): - """Calculate relative qualities of receivers and devices.""" - - if logger is None: - logger = current_app.logger - - # Calculate avg quality of devices - dev_sq = session.query(RelationStats.device_id, func.avg(RelationStats.quality).label("quality")).filter(RelationStats.date == date).group_by(RelationStats.device_id).subquery() - - dev_upd = update(DeviceStats).where(and_(DeviceStats.date == date, DeviceStats.device_id == dev_sq.c.device_id)).values({"quality": dev_sq.c.quality}) - - dev_result = session.execute(dev_upd) - dev_update_counter = dev_result.rowcount - session.commit() - logger.warn("Updated {} DeviceStats: quality".format(dev_update_counter)) - - # Calculate avg quality of receivers - rec_sq = session.query(RelationStats.receiver_id, func.avg(RelationStats.quality).label("quality")).filter(RelationStats.date == date).group_by(RelationStats.receiver_id).subquery() - - rec_upd = update(ReceiverStats).where(and_(ReceiverStats.date == date, ReceiverStats.receiver_id == rec_sq.c.receiver_id)).values({"quality": rec_sq.c.quality}) - - rec_result = session.execute(rec_upd) - rec_update_counter = rec_result.rowcount - session.commit() - logger.warn("Updated {} ReceiverStats: quality".format(rec_update_counter)) - - # Calculate quality_offset of devices - dev_sq = ( - session.query( - RelationStats.device_id, (func.sum(RelationStats.beacon_count * (RelationStats.quality - ReceiverStats.quality)) / (func.sum(RelationStats.beacon_count))).label("quality_offset") - ) - .filter(RelationStats.date == date) - .filter(and_(RelationStats.receiver_id == ReceiverStats.receiver_id, RelationStats.date == ReceiverStats.date)) - .group_by(RelationStats.device_id) - .subquery() - ) - - dev_upd = update(DeviceStats).where(and_(DeviceStats.date == date, DeviceStats.device_id == dev_sq.c.device_id)).values({"quality_offset": dev_sq.c.quality_offset}) - - dev_result = session.execute(dev_upd) - dev_update_counter = dev_result.rowcount - session.commit() - logger.warn("Updated {} DeviceStats: quality_offset".format(dev_update_counter)) - - # Calculate quality_offset of receivers - rec_sq = ( - session.query( - RelationStats.receiver_id, (func.sum(RelationStats.beacon_count * (RelationStats.quality - DeviceStats.quality)) / (func.sum(RelationStats.beacon_count))).label("quality_offset") - ) - .filter(RelationStats.date == date) - .filter(and_(RelationStats.device_id == DeviceStats.device_id, RelationStats.date == DeviceStats.date)) - .group_by(RelationStats.receiver_id) - .subquery() - ) - - rec_upd = update(ReceiverStats).where(and_(ReceiverStats.date == date, ReceiverStats.receiver_id == rec_sq.c.receiver_id)).values({"quality_offset": rec_sq.c.quality_offset}) - - rec_result = session.execute(rec_upd) - rec_update_counter = rec_result.rowcount - session.commit() - logger.warn("Updated {} ReceiverStats: quality_offset".format(rec_update_counter)) - - return "Updated {} DeviceStats and {} ReceiverStats".format(dev_update_counter, rec_update_counter) - - -def update_receivers(session, logger=None): - """Update receivers with stats.""" - - if logger is None: - logger = current_app.logger - - receiver_stats = ( - session.query( - distinct(ReceiverStats.receiver_id).label("receiver_id"), - func.first_value(ReceiverStats.firstseen) - .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.firstseen == null(), None)], else_=ReceiverStats.date).asc().nullslast()) - .label("firstseen"), - func.first_value(ReceiverStats.lastseen) - .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.lastseen == null(), None)], else_=ReceiverStats.date).desc().nullslast()) - .label("lastseen"), - func.first_value(ReceiverStats.location_wkt) - .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.location_wkt == null(), None)], else_=ReceiverStats.date).desc().nullslast()) - .label("location_wkt"), - func.first_value(ReceiverStats.altitude) - .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.altitude == null(), None)], else_=ReceiverStats.date).desc().nullslast()) - .label("altitude"), - func.first_value(ReceiverStats.version) - .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.version == null(), None)], else_=ReceiverStats.date).desc().nullslast()) - .label("version"), - func.first_value(ReceiverStats.platform) - .over(partition_by=ReceiverStats.receiver_id, order_by=case([(ReceiverStats.platform == null(), None)], else_=ReceiverStats.date).desc().nullslast()) - .label("platform"), - ) - .order_by(ReceiverStats.receiver_id) - .subquery() - ) - - upd = ( - update(Receiver) - .where(and_(Receiver.id == receiver_stats.c.receiver_id)) - .values( - { - "firstseen": receiver_stats.c.firstseen, - "lastseen": receiver_stats.c.lastseen, - "location": receiver_stats.c.location_wkt, - "altitude": receiver_stats.c.altitude, - "version": receiver_stats.c.version, - "platform": receiver_stats.c.platform, - } - ) - ) - - result = session.execute(upd) - update_counter = result.rowcount - session.commit() - logger.warn("Updated {} Receivers".format(update_counter)) - - return "Updated {} Receivers".format(update_counter) - - -def update_devices(session, logger=None): - """Update devices with stats.""" - - if logger is None: - logger = current_app.logger - - device_stats = ( - session.query( - distinct(DeviceStats.device_id).label("device_id"), - func.first_value(DeviceStats.name).over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.name == null(), None)], else_=DeviceStats.date).desc().nullslast()).label("name"), - func.first_value(DeviceStats.firstseen) - .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.firstseen == null(), None)], else_=DeviceStats.date).asc().nullslast()) - .label("firstseen"), - func.max(DeviceStats.lastseen) - .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.lastseen == null(), None)], else_=DeviceStats.date).desc().nullslast()) - .label("lastseen"), - func.first_value(DeviceStats.aircraft_type) - .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.aircraft_type == null(), None)], else_=DeviceStats.date).desc().nullslast()) - .label("aircraft_type"), - func.first_value(DeviceStats.stealth) - .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.stealth == null(), None)], else_=DeviceStats.date).desc().nullslast()) - .label("stealth"), - func.first_value(DeviceStats.software_version) - .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.software_version == null(), None)], else_=DeviceStats.date).desc().nullslast()) - .label("software_version"), - func.first_value(DeviceStats.hardware_version) - .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.hardware_version == null(), None)], else_=DeviceStats.date).desc().nullslast()) - .label("hardware_version"), - func.first_value(DeviceStats.real_address) - .over(partition_by=DeviceStats.device_id, order_by=case([(DeviceStats.real_address == null(), None)], else_=DeviceStats.date).desc().nullslast()) - .label("real_address"), - ) - .order_by(DeviceStats.device_id) - .subquery() - ) - - upd = ( - update(Device) - .where(and_(Device.id == device_stats.c.device_id)) - .values( - { - "name": device_stats.c.name, - "firstseen": device_stats.c.firstseen, - "lastseen": device_stats.c.lastseen, - "aircraft_type": device_stats.c.aircraft_type, - "stealth": device_stats.c.stealth, - "software_version": device_stats.c.software_version, - "hardware_version": device_stats.c.hardware_version, - "real_address": device_stats.c.real_address, - } - ) - ) - - result = session.execute(upd) - update_counter = result.rowcount - session.commit() - logger.warn("Updated {} Devices".format(update_counter)) - - return "Updated {} Devices".format(update_counter) diff --git a/app/collect/takeoff_landings.py b/app/collect/takeoff_landings.py deleted file mode 100644 index 0894184..0000000 --- a/app/collect/takeoff_landings.py +++ /dev/null @@ -1,146 +0,0 @@ -from datetime import timedelta - -from flask import current_app -from sqlalchemy import and_, or_, insert, between, exists -from sqlalchemy.sql import func, null -from sqlalchemy.sql.expression import case - -from app.model import AircraftBeacon, Device, TakeoffLanding, Airport - - -def update_entries(session, start, end, logger=None): - """Compute takeoffs and landings.""" - - if logger is None: - logger = current_app.logger - - logger.info("Compute takeoffs and landings.") - - # considered time interval should not exceed a complete day - if end - start > timedelta(days=1): - abort_message = "TakeoffLanding: timeinterval start='{}' and end='{}' is too big.".format(start, end) - logger.warn(abort_message) - return abort_message - - # check if we have any airport - airports_query = session.query(Airport).limit(1) - if not airports_query.all(): - abort_message = "TakeoffLanding: Cannot calculate takeoff and landings without any airport! Please import airports first." - logger.warn(abort_message) - return abort_message - - # delete existing elements - session.query(TakeoffLanding)\ - .filter(between(TakeoffLanding.timestamp, start, end))\ - .delete(synchronize_session='fetch') - session.commit() - - # takeoff / landing detection is based on 3 consecutive points all below a certain altitude AGL - takeoff_speed = 55 # takeoff detection: 1st point below, 2nd and 3rd above this limit - landing_speed = 40 # landing detection: 1st point above, 2nd and 3rd below this limit - min_takeoff_climb_rate = -5 # takeoff detection: glider should not sink too much - max_landing_climb_rate = 5 # landing detection: glider should not climb too much - duration = 100 # the points must not exceed this duration - radius = 5000 # the points must not exceed this radius around the 2nd point - max_agl = 200 # takeoff / landing must not exceed this altitude AGL - - # get beacons for selected time range (+ buffer for duration), one per address and timestamp - sq = ( - session.query(AircraftBeacon) - .distinct(AircraftBeacon.address, AircraftBeacon.timestamp) - .order_by(AircraftBeacon.address, AircraftBeacon.timestamp, AircraftBeacon.error_count) - .filter(AircraftBeacon.agl <= max_agl) - .filter(between(AircraftBeacon.timestamp, start - timedelta(seconds=duration), end + timedelta(seconds=duration))) - .subquery() - ) - - # make a query with current, previous and next position - sq2 = session.query( - sq.c.address, - func.lag(sq.c.address).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("address_prev"), - func.lead(sq.c.address).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("address_next"), - sq.c.timestamp, - func.lag(sq.c.timestamp).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("timestamp_prev"), - func.lead(sq.c.timestamp).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("timestamp_next"), - sq.c.location, - func.lag(sq.c.location).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("location_wkt_prev"), - func.lead(sq.c.location).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("location_wkt_next"), - sq.c.track, - func.lag(sq.c.track).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("track_prev"), - func.lead(sq.c.track).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("track_next"), - sq.c.ground_speed, - func.lag(sq.c.ground_speed).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("ground_speed_prev"), - func.lead(sq.c.ground_speed).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("ground_speed_next"), - sq.c.altitude, - func.lag(sq.c.altitude).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("altitude_prev"), - func.lead(sq.c.altitude).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("altitude_next"), - sq.c.climb_rate, - func.lag(sq.c.climb_rate).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("climb_rate_prev"), - func.lead(sq.c.climb_rate).over(partition_by=sq.c.address, order_by=sq.c.timestamp).label("climb_rate_next"), - ).subquery() - - # consider only positions between start and end and with predecessor and successor and limit distance and duration between points - sq3 = ( - session.query(sq2) - .filter(and_(sq2.c.address_prev != null(), sq2.c.address_next != null())) - .filter(and_(func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_prev) < radius, func.ST_DistanceSphere(sq2.c.location, sq2.c.location_wkt_next) < radius)) - .filter(sq2.c.timestamp_next - sq2.c.timestamp_prev < timedelta(seconds=duration)) - .filter(between(sq2.c.timestamp, start, end)) - .subquery() - ) - - # find possible takeoffs and landings - sq4 = ( - session.query( - sq3.c.timestamp, - case( - [ - (sq3.c.ground_speed > takeoff_speed, sq3.c.location_wkt_prev), # on takeoff we take the location from the previous fix because it is nearer to the airport - (sq3.c.ground_speed <= takeoff_speed, sq3.c.location), - ] - ).label("location"), - case([(sq3.c.ground_speed > landing_speed, sq3.c.track), (sq3.c.ground_speed <= landing_speed, sq3.c.track_prev)]).label( - "track" - ), # on landing we take the track from the previous fix because gliders tend to leave the runway quickly - sq3.c.ground_speed, - sq3.c.altitude, - case([(sq3.c.ground_speed > takeoff_speed, True), (sq3.c.ground_speed < landing_speed, False)]).label("is_takeoff"), - sq3.c.address, - ) - .filter( - or_( - and_(sq3.c.ground_speed_prev < takeoff_speed, sq3.c.ground_speed > takeoff_speed, sq3.c.ground_speed_next > takeoff_speed, sq3.c.climb_rate > min_takeoff_climb_rate), # takeoff - and_(sq3.c.ground_speed_prev > landing_speed, sq3.c.ground_speed < landing_speed, sq3.c.ground_speed_next < landing_speed, sq3.c.climb_rate < max_landing_climb_rate), # landing - ) - ) - .subquery() - ) - - # get the device id instead of the address and consider them if the are near airports ... - sq5 = ( - session.query( - sq4.c.timestamp, sq4.c.track, sq4.c.is_takeoff, sq4.c.address, Airport.id.label("airport_id"), func.ST_DistanceSphere(sq4.c.location, Airport.location_wkt).label("airport_distance") - ) - .filter(and_(func.ST_Within(sq4.c.location, Airport.border), - between(Airport.style, 2, 5))) - .subquery() - ) - - # ... and take the nearest airport - takeoff_landing_query = ( - session.query(sq5.c.timestamp, sq5.c.track, sq5.c.is_takeoff, sq5.c.address, sq5.c.airport_id) - .distinct(sq5.c.timestamp, sq5.c.track, sq5.c.is_takeoff, sq5.c.address) - .order_by(sq5.c.timestamp, sq5.c.track, sq5.c.is_takeoff, sq5.c.address, sq5.c.airport_distance) - .subquery() - ) - - # ... and save them - ins = insert(TakeoffLanding).from_select((TakeoffLanding.timestamp, TakeoffLanding.track, TakeoffLanding.is_takeoff, TakeoffLanding.address, TakeoffLanding.airport_id), takeoff_landing_query) - - result = session.execute(ins) - session.commit() - insert_counter = result.rowcount - - finish_message = "TakeoffLandings: {} inserted".format(insert_counter) - logger.info(finish_message) - return finish_message diff --git a/app/collect/timescaledb_views.py b/app/collect/timescaledb_views.py new file mode 100644 index 0000000..b8d5411 --- /dev/null +++ b/app/collect/timescaledb_views.py @@ -0,0 +1,165 @@ +from app import db +from app.utils import get_sql_trustworthy + +SQL_TRUSTWORTHY = get_sql_trustworthy(source_table_alias='sp') + +def create_views(): + db.session.execute(f""" + DROP VIEW IF EXISTS receiver_ranking CASCADE; + + CREATE VIEW receiver_ranking AS + SELECT + r.name AS receiver_name, + r.id AS receiver_id, + MAX(rs.max_distance) AS max_distance, + SUM(rs.max_normalized_quality * rs.messages_count) / SUM(rs.messages_count) AS max_normalized_quality, + SUM(rs.messages_count) AS messages_count, + COUNT(DISTINCT rs.sender_id) AS senders_count, + COUNT(DISTINCT rs.location_mgrs_short) AS coverage_count + FROM coverage_statistics AS rs + INNER JOIN receivers AS r ON rs.receiver_id = r.id + WHERE rs.date = NOW()::date AND rs.is_trustworthy IS TRUE AND rs.max_distance IS NOT NULL + GROUP BY rs.date, r.name, r.id + ORDER BY max_distance DESC; + """) + + db.session.execute(f""" + DROP VIEW IF EXISTS sender_ranking CASCADE; + + CREATE VIEW sender_ranking AS + SELECT + s.name, + s.id AS sender_id, + MAX(rs.max_distance) AS max_distance, + SUM(rs.max_normalized_quality * rs.messages_count) / SUM(rs.messages_count) AS max_normalized_quality, + SUM(rs.messages_count) AS messages_count, + COUNT(DISTINCT rs.receiver_id) AS receivers_count, + COUNT(DISTINCT rs.location_mgrs_short) AS coverage_count + FROM coverage_statistics AS rs + INNER JOIN senders AS s ON rs.sender_id = s.id + WHERE rs.date = NOW()::date AND rs.is_trustworthy IS TRUE AND rs.max_distance IS NOT NULL + GROUP BY rs.date, s.name, s.id + ORDER BY max_distance DESC; + """) + + db.session.commit() + +def create_timescaledb_views(): + # 1. Since the reference_timestamps are strictly increasing we can set + # the parameter 'refresh_lag' to a very short time so the materialization + # starts right after the bucket is finished + # 2. The feature realtime aggregation from TimescaleDB is quite time consuming. + # So we set materialized_only=true + + ### Sender statistics + # These stats will be used in the daily ranking, so we make the bucket < 1d + db.session.execute(f""" + DROP VIEW IF EXISTS sender_stats_1h CASCADE; + + CREATE VIEW sender_stats_1h + WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.refresh_lag='5 minutes') AS + SELECT + time_bucket(INTERVAL '1 hour', sp.reference_timestamp) AS bucket, + sp.name, + ({SQL_TRUSTWORTHY}) AS is_trustworthy, + COUNT(sp.*) AS beacon_count, + MAX(sp.distance) AS max_distance, + MIN(sp.altitude) AS min_altitude, + MAX(sp.altitude) AS max_altitude + + FROM sender_positions AS sp + GROUP BY bucket, sp.name, is_trustworthy; + """) + + # ... and just for curiosity also bucket = 1d + db.session.execute(f""" + DROP VIEW IF EXISTS sender_stats_1d CASCADE; + + CREATE VIEW sender_stats_1d + WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.refresh_lag='1 hour') AS + SELECT + time_bucket(INTERVAL '1 day', sp.reference_timestamp) AS bucket, + sp.name, + ({SQL_TRUSTWORTHY}) AS is_trustworthy, + COUNT(sp.*) AS beacon_count, + MAX(sp.distance) AS max_distance, + MIN(sp.altitude) AS min_altitude, + MAX(sp.altitude) AS max_altitude + + FROM sender_positions AS sp + GROUP BY bucket, sp.name, is_trustworthy; + """) + + ### Receiver statistics + # These stats will be used in the daily ranking, so we make the bucket < 1d + db.session.execute(f""" + DROP VIEW IF EXISTS receiver_stats_1h CASCADE; + + CREATE VIEW receiver_stats_1h + WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.refresh_lag='5 minutes') AS + SELECT + time_bucket(INTERVAL '1 hour', sp.reference_timestamp) AS bucket, + sp.receiver_name, + ({SQL_TRUSTWORTHY}) AS is_trustworthy, + COUNT(sp.*) AS beacon_count, + MAX(sp.distance) AS max_distance, + MIN(sp.altitude) AS min_altitude, + MAX(sp.altitude) AS max_altitude + + FROM sender_positions AS sp + GROUP BY bucket, sp.receiver_name, is_trustworthy; + """) + + # ... and just for curiosity also bucket = 1d + db.session.execute(f""" + DROP VIEW IF EXISTS receiver_stats_1d CASCADE; + + CREATE VIEW receiver_stats_1d + WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.refresh_lag='1 hour') AS + SELECT + time_bucket(INTERVAL '1 day', sp.reference_timestamp) AS bucket, + sp.receiver_name, + ({SQL_TRUSTWORTHY}) AS is_trustworthy, + COUNT(sp.*) AS beacon_count, + MAX(sp.distance) AS max_distance, + MIN(sp.altitude) AS min_altitude, + MAX(sp.altitude) AS max_altitude + + FROM sender_positions AS sp + GROUP BY bucket, sp.receiver_name, is_trustworthy; + """) + + ### Relation statistics (sender <-> receiver) + # these stats will be used on a >= 1d basis, so we make the bucket = 1d + db.session.execute(f""" + DROP VIEW IF EXISTS relation_stats_1d CASCADE; + + CREATE VIEW relation_stats_1d + WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.refresh_lag='1 hour') AS + SELECT + time_bucket(INTERVAL '1 day', sp.reference_timestamp) AS bucket, + sp.name, + sp.receiver_name, + ({SQL_TRUSTWORTHY}) AS is_trustworthy, + COUNT(sp.*) AS beacon_count, + MAX(sp.normalized_quality) AS max_normalized_quality, + MAX(sp.distance) AS max_distance + + FROM sender_positions AS sp + GROUP BY bucket, sp.name, sp.receiver_name, is_trustworthy; + """) + + db.session.commit() + + +""" +class MyView(db.Model): + __table__ = db.Table( + 'device_stats', db.metadata, + db.Column('bucket', db.DateTime, primary_key=True), + db.Column('name', db.String, primary_key=True), + db.Column('beacon_count', db.Integer), + autoload=True, + autoload_with=db.engine + ) +""" \ No newline at end of file diff --git a/app/commands/database.py b/app/commands/database.py index 9f1c550..cadd86a 100644 --- a/app/commands/database.py +++ b/app/commands/database.py @@ -5,9 +5,10 @@ import click from datetime import datetime from sqlalchemy.sql import func -from app.collect.database import update_device_infos, update_country_code -from app.model import AircraftBeacon, DeviceInfoOrigin +from app.collect.database import update_device_infos +from app.model import SenderPosition, SenderInfoOrigin from app.utils import get_airports, get_days +from app.collect.timescaledb_views import create_timescaledb_views, create_views from app import db @@ -22,7 +23,7 @@ def get_database_days(start, end): """Returns the first and the last day in aircraft_beacons table.""" if start is None and end is None: - days_from_db = db.session.query(func.min(AircraftBeacon.timestamp).label("first_day"), func.max(AircraftBeacon.timestamp).label("last_day")).one() + days_from_db = db.session.query(func.min(SenderPosition.timestamp).label("first_day"), func.max(SenderPosition.timestamp).label("last_day")).one() start = days_from_db[0].date() end = days_from_db[1].date() else: @@ -60,8 +61,8 @@ def init_timescaledb(): """Initialize TimescaleDB features.""" db.session.execute("CREATE EXTENSION IF NOT EXISTS timescaledb;") - db.session.execute("SELECT create_hypertable('aircraft_beacons', 'timestamp', chunk_time_interval => interval '6 hours', if_not_exists => TRUE);") - db.session.execute("SELECT create_hypertable('receiver_beacons', 'timestamp', chunk_time_interval => interval '6 hours', if_not_exists => TRUE);") + db.session.execute("SELECT create_hypertable('sender_positions', 'reference_timestamp', chunk_time_interval => interval '3 hours', if_not_exists => TRUE);") + db.session.execute("SELECT create_hypertable('receiver_positions', 'reference_timestamp', chunk_time_interval => interval '1 day', if_not_exists => TRUE);") db.session.commit() print("Done.") @@ -83,7 +84,7 @@ def import_ddb(): """Import registered devices from the DDB.""" print("Import registered devices fom the DDB...") - counter = update_device_infos(db.session, DeviceInfoOrigin.OGN_DDB) + counter = update_device_infos(SenderInfoOrigin.OGN_DDB) print("Imported %i devices." % counter) @@ -93,7 +94,7 @@ def import_file(path="tests/custom_ddb.txt"): """Import registered devices from a local file.""" print("Import registered devices from '{}'...".format(path)) - counter = update_device_infos(db.session, DeviceInfoOrigin.USER_DEFINED, path=path) + counter = update_device_infos(SenderInfoOrigin.USER_DEFINED, path=path) print("Imported %i devices." % counter) @@ -103,7 +104,7 @@ def import_flarmnet(path=None): """Import registered devices from a local file.""" print("Import registered devices from '{}'...".format("internet" if path is None else path)) - counter = update_device_infos(db.session, DeviceInfoOrigin.FLARMNET, path=path) + counter = update_device_infos(SenderInfoOrigin.FLARMNET, path=path) print("Imported %i devices." % counter) @@ -116,13 +117,23 @@ def import_airports(path="tests/SeeYou.cup"): airports = get_airports(path) db.session.bulk_save_objects(airports) db.session.commit() - db.session.execute("UPDATE airports SET border = ST_Expand(location, 0.05)") + # TODO: SRID 4087 ist nicht korrekt, aber spherical mercator 3857 wirft hier Fehler + db.session.execute("UPDATE airports AS a SET border = ST_Transform(ST_Buffer(ST_Transform(location, 4087), 1.5 * GREATEST(500, a.runway_length)), 4326);") db.session.commit() print("Imported {} airports.".format(len(airports))) +@user_cli.command("create_timescaledb_views") +def cmd_create_timescaledb_views(): + """Create TimescaleDB views.""" + + create_timescaledb_views() + print("Done") + +@user_cli.command("create_views") +def cmd_create_views(): + """Create views.""" + + create_views() + print("Done") -@user_cli.command("update_country_codes") -def update_country_codes(): - """Update country codes of all receivers.""" - update_country_code(session=db.session) diff --git a/app/commands/export.py b/app/commands/export.py index 11f0756..7f5d831 100644 --- a/app/commands/export.py +++ b/app/commands/export.py @@ -4,14 +4,86 @@ import click import datetime import re import csv +import os from aerofiles.igc import Writer -from app.model import AircraftBeacon, Device +from app.model import SenderPosition, Sender from app import db user_cli = AppGroup("export") user_cli.help = "Export data in several file formats." +@user_cli.command("debug_sql") +@click.argument("start") +@click.argument("end") +@click.argument("name") +def debug_sql(start, end, name): + """Export data (sender_positions and receivers) as sql for debugging (and/or creating test cases).""" + + # First: get all the positions (and the receiver names for later) + sql_sender_positions = f""" + SELECT reference_timestamp, name, receiver_name, timestamp, location, track, ground_speed, altitude, aircraft_type, climb_rate, turn_rate, distance, bearing, agl + FROM sender_positions + WHERE reference_timestamp BETWEEN '{start}' AND '{end}' AND name = '{name}' + ORDER BY reference_timestamp; + """ + + receiver_names = [] + sender_position_values = [] + results = db.session.execute(sql_sender_positions) + for row in results: + if row[2] not in receiver_names: + receiver_names.append("'" + row[2] + "'") + row = [f"'{r}'" if r else "DEFAULT" for r in row] + sender_position_values.append(f"({','.join(row)})") + + # Second: get the receivers + sql_receivers = f""" + SELECT name, location + FROM receivers + WHERE name IN ({','.join(receiver_names)}); + """ + + receiver_values = [] + results = db.session.execute(sql_receivers) + for row in results: + row = [f"'{r}'" if r else "DEFAULT" for r in row] + receiver_values.append(f"({','.join(row)})") + + # Third: get the airports + sql_airports = f""" + SELECT DISTINCT a.name, a.location, a.altitude, a.style, a.border + FROM airports AS a, receivers AS r + WHERE + r.name IN ({','.join(receiver_names)}) + AND ST_Within(r.location, ST_Buffer(a.location, 0.2)) + AND a.style IN (2,4,5); + """ + + airport_values = [] + results = db.session.execute(sql_airports) + for row in results: + row = [f"'{r}'" if r else "DEFAULT" for r in row] + airport_values.append(f"({','.join(row)})") + + # Last: write all into file + with open(f'{start}_{end}_{name}.sql', 'w') as file: + file.write(f'/*\n') + file.write(f'OGN Python SQL Export\n') + file.write(f'Created by: {os.getlogin()}\n') + file.write(f'Created at: {datetime.datetime.utcnow()}\n') + file.write(f'*/\n\n') + + + file.write("INSERT INTO airports(name, location, altitude, style, border) VALUES\n") + file.write(',\n'.join(airport_values) + ';\n\n') + + file.write("INSERT INTO receivers(name, location) VALUES\n") + file.write(',\n'.join(receiver_values) + ';\n\n') + + file.write("INSERT INTO sender_positions(reference_timestamp, name, receiver_name, timestamp, location, track, ground_speed, altitude, aircraft_type, climb_rate, turn_rate, distance, bearing, agl) VALUES\n") + file.write(',\n'.join(sender_position_values) + ';\n\n') + @user_cli.command("cup") def cup(): @@ -68,7 +140,7 @@ def igc(address, date): print("Date {} not valid.".format(date)) return - device_id = db.session.query(Device.id).filter(Device.address == address).first() + device_id = db.session.query(Sender.id).filter(Sender.address == address).first() if device_id is None: print("Device with address '{}' not found.".format(address)) @@ -98,11 +170,11 @@ def igc(address, date): ) points = ( - db.session.query(AircraftBeacon) - .filter(AircraftBeacon.device_id == device_id) - .filter(AircraftBeacon.timestamp > date + " 00:00:00") - .filter(AircraftBeacon.timestamp < date + " 23:59:59") - .order_by(AircraftBeacon.timestamp) + db.session.query(SenderPosition) + .filter(SenderPosition.device_id == device_id) + .filter(SenderPosition.timestamp > date + " 00:00:00") + .filter(SenderPosition.timestamp < date + " 23:59:59") + .order_by(SenderPosition.timestamp) ) for point in points.all(): diff --git a/app/commands/flights.py b/app/commands/flights.py index 5c81f63..035546c 100644 --- a/app/commands/flights.py +++ b/app/commands/flights.py @@ -6,119 +6,11 @@ from tqdm import tqdm from app.commands.database import get_database_days from app import db +from app.collect.flights import compute_flights, compute_gaps user_cli = AppGroup("flights") user_cli.help = "Create 2D flight paths from data." -NOTHING = "" -CONTEST_RELEVANT = "AND agl < 1000" -LOW_PASS = "AND agl < 50 and ground_speed > 250" - - -def compute_gaps(session, date): - query = """ - INSERT INTO flights2d(date, flight_type, device_id, path) - SELECT '{date}' AS date, - 3 AS flight_type, - sq3.device_id, - ST_Collect(sq3.path) - FROM ( - SELECT sq2.d1 device_id, - ST_MakeLine(sq2.l1, sq2.l2) path - FROM - ( - SELECT sq.timestamp t1, - LAG(sq.timestamp) OVER ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) t2, - sq.location l1, - LAG(sq.location) OVER ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) l2, - sq.device_id d1, - LAG(sq.device_id) OVER ( PARTITION BY sq.timestamp::DATE, sq.device_id ORDER BY sq.timestamp) d2 - FROM - ( - SELECT DISTINCT ON (device_id, timestamp) timestamp, device_id, location, agl - FROM aircraft_beacons - WHERE timestamp BETWEEN '{date} 00:00:00' AND '{date} 23:59:59' AND agl > 300 - ORDER BY device_id, timestamp, error_count - ) sq - ) sq2 - WHERE EXTRACT(epoch FROM sq2.t1 - sq2.t2) > 300 - AND ST_DistanceSphere(sq2.l1, sq2.l2) / EXTRACT(epoch FROM sq2.t1 - sq2.t2) BETWEEN 15 AND 50 - ) sq3 - GROUP BY sq3.device_id - ON CONFLICT DO NOTHING; - """.format( - date=date.strftime("%Y-%m-%d") - ) - - session.execute(query) - session.commit() - - -def compute_flights2d(session, date, flight_type): - if flight_type == 0: - filter = NOTHING - elif flight_type == 1: - filter = CONTEST_RELEVANT - elif flight_type == 2: - filter = LOW_PASS - - query = """ - INSERT INTO flights2d - ( - date, - flight_type, - device_id, - path, - path_simple - ) - SELECT '{date}' AS date, - {flight_type} as flight_type, - sq5.device_id, - st_collect(sq5.linestring order BY sq5.part) multilinestring, - st_collect(st_simplify(sq5.linestring, 0.0001) ORDER BY sq5.part) simple_multilinestring - FROM ( - SELECT sq4.device_id, - sq4.part, - st_makeline(sq4.location ORDER BY sq4.timestamp) linestring - FROM ( - SELECT sq3.timestamp, - sq3.location, - sq3.device_id, - sum(sq3.ping) OVER (partition BY sq3.device_id ORDER BY sq3.timestamp) part - FROM ( - SELECT sq2.t1 AS timestamp, - sq2.l1 AS location, - sq2.d1 device_id, - CASE - WHEN sq2.t1 - sq2.t2 < interval'100s' AND ST_DistanceSphere(sq2.l1, sq2.l2) < 1000 THEN 0 - ELSE 1 - END AS ping - FROM ( - SELECT sq.timestamp t1, - lag(sq.timestamp) OVER (partition BY sq.device_id ORDER BY sq.timestamp) t2, - sq.location l1, - lag(sq.location) OVER (partition BY sq.device_id ORDER BY sq.timestamp) l2, - sq.device_id d1, - lag(sq.device_id) OVER (partition BY sq.device_id ORDER BY sq.timestamp) d2 - FROM ( - SELECT DISTINCT ON (device_id, timestamp) timestamp, device_id, location - FROM aircraft_beacons - WHERE timestamp BETWEEN '{date} 00:00:00' AND '{date} 23:59:59' {filter} - ORDER BY device_id, timestamp, error_count - ) sq - ) sq2 - ) sq3 - ) sq4 - GROUP BY sq4.device_id, sq4.part - ) sq5 - GROUP BY sq5.device_id - ON CONFLICT DO NOTHING; - """.format( - date=date.strftime("%Y-%m-%d"), flight_type=flight_type, filter=filter - ) - session.execute(query) - session.commit() - @user_cli.command("create") @click.argument("start") @@ -133,6 +25,6 @@ def create(start, end, flight_type): for single_date in pbar: pbar.set_description(datetime.strftime(single_date, "%Y-%m-%d")) if flight_type <= 2: - result = compute_flights2d(session=db.session, date=single_date, flight_type=flight_type) + result = compute_flights(date=single_date, flight_type=flight_type) else: - result = compute_gaps(session=db.session, date=single_date) + result = compute_gaps(date=single_date) diff --git a/app/commands/gateway.py b/app/commands/gateway.py index 8ea9505..65701e3 100644 --- a/app/commands/gateway.py +++ b/app/commands/gateway.py @@ -1,5 +1,6 @@ import os from datetime import datetime, timezone +import time from flask import current_app from flask.cli import AppGroup @@ -7,39 +8,66 @@ import click from tqdm import tqdm from ogn.client import AprsClient +from ogn.parser import parse from app import redis_client -from app.gateway.bulkimport import convert, calculate +from app.gateway.beacon_conversion import aprs_string_to_message +from app.gateway.message_handling import receiver_status_message_to_csv_string, receiver_position_message_to_csv_string, sender_position_message_to_csv_string +from app.collect.gateway import transfer_from_redis_to_database user_cli = AppGroup("gateway") user_cli.help = "Connection to APRS servers." @user_cli.command("run") -def run(aprs_user="anon-dev"): - """Run the aprs client and feed the redis db with incoming data.""" - - # User input validation - if len(aprs_user) < 3 or len(aprs_user) > 9: - print("aprs_user must be a string of 3-9 characters.") - return +@click.option("--aprs_filter", default='') +def run(aprs_filter): + """ + Run the aprs client, parse the incoming data and put it to redis. + """ current_app.logger.warning("Start ogn gateway") - client = AprsClient(aprs_user) + client = AprsClient(current_app.config['APRS_USER'], aprs_filter) client.connect() def insert_into_redis(aprs_string): - redis_client.set(f"ogn-python {datetime.utcnow()}", aprs_string.strip(), ex=100) + # Convert aprs_string to message dict, add MGRS Position, flatten gps precision, etc. etc. ... + message = aprs_string_to_message(aprs_string) + if message is None: + return + + # separate between tables (receiver/sender) and aprs_type (status/position) + if message['beacon_type'] in ('aprs_receiver', 'receiver'): + if message['aprs_type'] == 'status': + redis_target = 'receiver_status' + csv_string = receiver_status_message_to_csv_string(message, none_character=r'\N') + elif message['aprs_type'] == 'position': + redis_target = 'receiver_position' + csv_string = receiver_position_message_to_csv_string(message, none_character=r'\N') + else: + return + else: + if message['aprs_type'] == 'status': + return # no interesting data we want to keep + elif message['aprs_type'] == 'position': + redis_target = 'sender_position' + csv_string = sender_position_message_to_csv_string(message, none_character=r'\N') + else: + return + + mapping = {csv_string: str(time.time())} + + redis_client.zadd(name=redis_target, mapping=mapping, nx=True) insert_into_redis.beacon_counter += 1 - - delta = (datetime.utcnow() - insert_into_redis.last_update).total_seconds() - if delta >= 60.0: - print(f"{insert_into_redis.beacon_counter/delta:05.1f}/s") - insert_into_redis.last_update = datetime.utcnow() + + current_minute = datetime.utcnow().minute + if current_minute != insert_into_redis.last_minute: + current_app.logger.warning(f"{insert_into_redis.beacon_counter:7d}") insert_into_redis.beacon_counter = 0 - + insert_into_redis.last_minute = current_minute + insert_into_redis.beacon_counter = 0 - insert_into_redis.last_update = datetime.utcnow() + insert_into_redis.last_minute = datetime.utcnow().minute try: client.run(callback=insert_into_redis, autoreconnect=True) @@ -48,12 +76,21 @@ def run(aprs_user="anon-dev"): client.disconnect() + +@user_cli.command("transfer") +def transfer(): + """Transfer data from redis to the database.""" + + transfer_from_redis_to_database() + + @user_cli.command("printout") -def printout(): +@click.option("--aprs_filter", default='') +def printout(aprs_filter): """Run the aprs client and just print out the data stream.""" - + current_app.logger.warning("Start ogn gateway") - client = AprsClient("anon-dev") + client = AprsClient(current_app.config['APRS_USER'], aprs_filter=aprs_filter) client.connect() try: diff --git a/app/commands/logbook.py b/app/commands/logbook.py index e34cf51..6e7daf4 100644 --- a/app/commands/logbook.py +++ b/app/commands/logbook.py @@ -3,18 +3,15 @@ import click from datetime import datetime -from app.collect.logbook import update_entries as logbook_update_entries -from app.collect.takeoff_landings import update_entries as takeoff_landings_update_entries +from app.collect.logbook import update_takeoff_landings, update_logbook from app.model import Airport, Logbook from sqlalchemy.sql import func from tqdm import tqdm from app.commands.database import get_database_days from app.utils import date_to_timestamps -from app import db - user_cli = AppGroup("logbook") -user_cli.help = "Handling of logbook data." +user_cli.help = "Handling of takeoff/landings and logbook data." @user_cli.command("compute_takeoff_landing") @@ -29,7 +26,7 @@ def compute_takeoff_landing(start, end): for single_date in pbar: pbar.set_description(datetime.strftime(single_date, "%Y-%m-%d")) (start, end) = date_to_timestamps(single_date) - result = takeoff_landings_update_entries(session=db.session, start=start, end=end) + result = update_takeoff_landings(start=start, end=end) @user_cli.command("compute_logbook") @@ -43,4 +40,4 @@ def compute_logbook(start, end): pbar = tqdm(days) for single_date in pbar: pbar.set_description(single_date.strftime("%Y-%m-%d")) - result = logbook_update_entries(session=db.session, date=single_date) + result = update_logbook(date=single_date) diff --git a/app/gateway/beacon_conversion.py b/app/gateway/beacon_conversion.py new file mode 100644 index 0000000..8421835 --- /dev/null +++ b/app/gateway/beacon_conversion.py @@ -0,0 +1,51 @@ +from mgrs import MGRS +import rasterio as rs + +from ogn.parser import parse + +from app.model import AircraftType + + +mgrs = MGRS() +#elevation_dataset = rs.open('/Volumes/LaCieBlack/Wtf4.tiff') + + +def aprs_string_to_message(aprs_string): + try: + message = parse(aprs_string, calculate_relations=True) + except Exception as e: + print(e) + return None + + if message['aprs_type'] not in ('position', 'status'): + return None + + elif message['aprs_type'] == 'position': + latitude = message["latitude"] + longitude = message["longitude"] + + message["location"] = "SRID=4326;POINT({} {})".format(longitude, latitude) + + location_mgrs = mgrs.toMGRS(latitude, longitude).decode("utf-8") + message["location_mgrs"] = location_mgrs + message["location_mgrs_short"] = location_mgrs[0:5] + location_mgrs[5:7] + location_mgrs[10:12] + + #if 'altitude' in message and longitude >= 0.0 and longitude <= 20.0 and latitude >= 40.0 and latitude <= 60.0: + # elevation = [val[0] for val in elevation_dataset.sample(((longitude, latitude),))][0] + # message['agl'] = message['altitude'] - elevation + + if 'bearing' in message: + bearing = int(message['bearing']) + message['bearing'] = bearing if bearing < 360 else 0 + + + if "aircraft_type" in message: + message["aircraft_type"] = AircraftType(message["aircraft_type"]) if message["aircraft_type"] in AircraftType.list() else AircraftType.UNKNOWN + + if "gps_quality" in message: + if message["gps_quality"] is not None and "horizontal" in message["gps_quality"]: + message["gps_quality_horizontal"] = message["gps_quality"]["horizontal"] + message["gps_quality_vertical"] = message["gps_quality"]["vertical"] + del message["gps_quality"] + + return message diff --git a/app/gateway/bulkimport.py b/app/gateway/bulkimport.py deleted file mode 100644 index fba9f6c..0000000 --- a/app/gateway/bulkimport.py +++ /dev/null @@ -1,571 +0,0 @@ -import os -import re -from datetime import datetime, timedelta -import time -from io import StringIO -import gzip - -from flask import current_app -from flask.cli import AppGroup -import click -from tqdm import tqdm -from mgrs import MGRS - -from ogn.parser import parse, ParseError - -from app.model import AircraftBeacon, ReceiverBeacon, AircraftType, Location -from app.gateway.process_tools import open_file - -from app import db - -user_cli = AppGroup("bulkimport") -user_cli.help = "Tools for accelerated data import." - - -basepath = os.path.dirname(os.path.realpath(__file__)) - -# define message types we want to proceed -AIRCRAFT_BEACON_TYPES = ["aprs_aircraft", "flarm", "tracker", "fanet", "lt24", "naviter", "pilot_aware", "skylines", "spider", "spot", "flymaster", "capturs"] -RECEIVER_BEACON_TYPES = ["aprs_receiver", "receiver"] - -# define fields we want to proceed -AIRCRAFT_POSITION_BEACON_FIELDS = [ - "location", - "altitude", - "name", - "dstcall", - "relay", - "receiver_name", - "timestamp", - "track", - "ground_speed", - "address_type", - "aircraft_type", - "stealth", - "address", - "climb_rate", - "turn_rate", - "signal_quality", - "error_count", - "frequency_offset", - "gps_quality_horizontal", - "gps_quality_vertical", - "software_version", - "hardware_version", - "real_address", - "signal_power", - "distance", - "radial", - "quality", - "location_mgrs", - "location_mgrs_short", - "agl", - - "reference_timestamp", -] - -RECEIVER_POSITION_BEACON_FIELDS = [ - "location", - "altitude", - "name", - "dstcall", - "receiver_name", - "timestamp", - - "reference_timestamp", -] - -RECEIVER_STATUS_BEACON_FIELDS = [ - "name", - "dstcall", - "receiver_name", - "timestamp", - - "version", - "platform", - - "reference_timestamp", -] - - -def initial_file_scan(file): - """Scan file and get rowcount and first server timestamp.""" - - row_count = 0 - timestamp = None - - for row in file: - row_count += 1 - if timestamp is None and row[0] == '#': - message = parse(row) - if message['aprs_type'] == 'server': - timestamp = message['timestamp'] - - file.seek(0) - return row_count, timestamp - - -class StringConverter: - mgrs = MGRS() - - def __enter__(self): - return self - - def __exit__(self, *args): - pass - - def _convert(self, raw_string, reference_timestamp): - try: - message = parse(raw_string, reference_timestamp) - except NotImplementedError as e: - current_app.logger.error("No parser implemented for message: {}".format(raw_string)) - return - except ParseError as e: - if not raw_string.startswith('RND'): # skip errors with RND since they are common - current_app.logger.error("Parsing error with message: {}".format(raw_string)) - - return - except TypeError as e: - current_app.logger.error("TypeError with message: {}".format(raw_string)) - return - except Exception as e: - current_app.logger.error("Other Exception with string: {}".format(raw_string)) - return - - if message['aprs_type'] not in ('position', 'status'): - return - - elif message['aprs_type'] == 'position': - latitude = message["latitude"] - longitude = message["longitude"] - - message["location"] = "SRID=4326;POINT({} {})".format(longitude, latitude) - - location_mgrs = self.mgrs.toMGRS(latitude, longitude).decode("utf-8") - message["location_mgrs"] = location_mgrs - message["location_mgrs_short"] = location_mgrs[0:5] + location_mgrs[5:7] + location_mgrs[10:12] - - if "aircraft_type" in message: - message["aircraft_type"] = AircraftType(message["aircraft_type"]) if message["aircraft_type"] in AircraftType.list() else AircraftType.UNKNOWN - - if "gps_quality" in message: - if message["gps_quality"] is not None and "horizontal" in message["gps_quality"]: - message["gps_quality_horizontal"] = message["gps_quality"]["horizontal"] - message["gps_quality_vertical"] = message["gps_quality"]["vertical"] - del message["gps_quality"] - - return message - - def _get_aircraft_position_beacon_csv_string(self, message, none_character=''): - csv_string = "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16},{17},{18},{19},{20},{21},{22},{23},{24},{25},{26},{27},{28},{29},{30}\n".format( - message['location'], - int(message['altitude']) if message['altitude'] else none_character, - message['name'], - message['dstcall'], - message['relay'] if 'relay' in message and message['relay'] else none_character, - message['receiver_name'], - message['timestamp'], - message['track'] if 'track' in message and message['track'] else none_character, - message['ground_speed'] if 'ground_speed' in message and message['ground_speed'] else none_character, - message['address_type'] if 'address_type' in message and message['address_type'] else none_character, - message['aircraft_type'].name if 'aircraft_type' in message and message['aircraft_type'] else AircraftType.UNKNOWN.name, - message['stealth'] if 'stealth' in message and message['stealth'] else none_character, - message['address'] if 'address' in message and message['address'] else none_character, - message['climb_rate'] if 'climb_rate' in message and message['climb_rate'] else none_character, - message['turn_rate'] if 'turn_rate' in message and message['turn_rate'] else none_character, - message['signal_quality'] if 'signal_quality' in message and message['signal_quality'] else none_character, - message['error_count'] if 'error_count' in message and message['error_count'] else none_character, - message['frequency_offset'] if 'frequency_offset' in message and message['frequency_offset'] else none_character, - message['gps_quality_horizontal'] if 'gps_quality_horizontal' in message and message['gps_quality_horizontal'] else none_character, - message['gps_quality_vertical'] if 'gps_quality_vertical' in message and message['gps_quality_vertical'] else none_character, - message['software_version'] if 'software_version' in message and message['software_version'] else none_character, #20 - message['hardware_version'] if 'hardware_version' in message and message['hardware_version'] else none_character, - message['real_address'] if 'real_address' in message and message['real_address'] else none_character, - message['signal_power'] if 'signal_power' in message and message['signal_power'] else none_character, - message['distance'] if 'distance' in message and message['distance'] else none_character, - message['radial'] if 'radial' in message and message['radial'] else none_character, - message['quality'] if 'quality' in message and message['quality'] else none_character, - message['location_mgrs'], - message['location_mgrs_short'], - message['agl'] if 'agl' in message else none_character, #29 - - message['reference_timestamp'], - ) - return csv_string - - def _get_receiver_position_beacon_csv_string(self, message, none_character=''): - csv_string = "{0},{1},{2},{3},{4},{5},{6}\n".format( - message['location'], - int(message['altitude']) if message['altitude'] else none_character, - message['name'], - message['dstcall'], - message['receiver_name'], - message['timestamp'], - - message['reference_timestamp'], - ) - return csv_string - - def _get_receiver_status_beacon_csv_string(self, message, none_character=''): - csv_string = "{0},{1},{2},{3},{4},{5},{6}\n".format( - message['name'], - message['dstcall'], - message['receiver_name'], - message['timestamp'], - - message['version'] if 'version' in message and message['version'] else none_character, - message['platform'] if 'platform' in message and message['platform'] else none_character, - - message['reference_timestamp'] - ) - return csv_string - - -class FileFeeder(StringConverter): - def __init__(self, postfix, reference_timestamp, reference_timestamp_autoupdate): - self.reference_timestamp = reference_timestamp - self.reference_timestamp_autoupdate = reference_timestamp_autoupdate - - self.aircraft_beacons_file = gzip.open('aircraft_beacons_{}.csv.gz'.format(postfix), 'wt') - self.receiver_beacons_file = gzip.open('receiver_beacons_{}.csv.gz'.format(postfix), 'wt') - - super().__init__(reference_timestamp, reference_timestamp_autoupdate) - - def __enter__(self): - self.aircraft_beacons_file.write(','.join(AIRCRAFT_POSITION_BEACON_FIELDS)) - self.receiver_beacons_file.write(','.join(RECEIVER_POSITION_BEACON_FIELDS)) - return self - - def __exit__(self, *args): - self.aircraft_beacons_file.close() - self.receiver_beacons_file.close() - - def add(self, raw_string): - message = self._convert(raw_string) - if message['beacon_type'] in AIRCRAFT_BEACON_TYPES: - csv_string = self._get_aircraft_position_beacon_csv_string(message) - self.aircraft_beacons_file.write(csv_string) - elif message['beacon_type'] in RECEIVER_BEACON_TYPES: - csv_string = self._get_receiver_position_beacon_csv_string(message) - self.receiver_beacons_file.write(csv_string) - - -class DbFeeder(StringConverter): - def __init__(self): - self.aircraft_position_beacons_buffer = StringIO() - self.aircraft_status_beacons_buffer = StringIO() - self.receiver_position_beacons_buffer = StringIO() - self.receiver_status_beacons_buffer = StringIO() - - self.last_flush = datetime.utcnow() - - def __exit__(self, *args): - self.flush() - - def add(self, raw_string, reference_timestamp): - raw_string = raw_string.strip() - - message = self._convert(raw_string, reference_timestamp=reference_timestamp) - if not message: - return - - message['reference_timestamp'] = reference_timestamp - if message['beacon_type'] in AIRCRAFT_BEACON_TYPES and message['aprs_type'] == 'position': - csv_string = self._get_aircraft_position_beacon_csv_string(message, none_character=r'\N') - self.aircraft_position_beacons_buffer.write(csv_string) - elif message['beacon_type'] in AIRCRAFT_BEACON_TYPES and message['aprs_type'] == 'status': - pass # ignore it - elif message['beacon_type'] in RECEIVER_BEACON_TYPES and message['aprs_type'] == 'position': - csv_string = self._get_receiver_position_beacon_csv_string(message, none_character=r'\N') - self.receiver_position_beacons_buffer.write(csv_string) - elif message['beacon_type'] in RECEIVER_BEACON_TYPES and message['aprs_type'] == 'status': - csv_string = self._get_receiver_status_beacon_csv_string(message, none_character=r'\N') - self.receiver_status_beacons_buffer.write(csv_string) - else: - current_app.logger.error(f"Not supported. beacon_type: '{message['beacon_type']}', aprs_type: '{message['aprs_type']}', skipped: {raw_string}") - - def _flush_position_beacons(self): - connection = db.engine.raw_connection() - cursor = connection.cursor() - - self.aircraft_position_beacons_buffer.seek(0) - self.receiver_position_beacons_buffer.seek(0) - - aircraft_position_beacons_temp_table_name = f"aircraft_position_beacons_temp_{str(time.time()).replace('.', '_')}" - receiver_position_beacons_temp_table_name = f"receiver_position_beacons_temp_{str(time.time()).replace('.', '_')}" - - cursor.execute(f"CREATE TEMPORARY TABLE {aircraft_position_beacons_temp_table_name} (LIKE aircraft_beacons) ON COMMIT DROP;") - cursor.execute(f"CREATE TEMPORARY TABLE {receiver_position_beacons_temp_table_name} (LIKE receiver_beacons) ON COMMIT DROP;") - - cursor.copy_from(file=self.aircraft_position_beacons_buffer, table=aircraft_position_beacons_temp_table_name, sep=",", columns=AIRCRAFT_POSITION_BEACON_FIELDS) - cursor.copy_from(file=self.receiver_position_beacons_buffer, table=receiver_position_beacons_temp_table_name, sep=",", columns=RECEIVER_POSITION_BEACON_FIELDS) - - # Update receivers - cursor.execute(f""" - INSERT INTO receivers AS r (name, location, altitude, firstseen, lastseen, timestamp) - SELECT DISTINCT ON (rpbt.name) - rpbt.name, - rpbt.location, - rpbt.altitude, - rpbt.reference_timestamp AS firstseen, - rpbt.reference_timestamp AS lastseen, - rpbt.timestamp - FROM {receiver_position_beacons_temp_table_name} AS rpbt, - ( - SELECT - rpbt.name, - MAX(timestamp) AS timestamp - FROM {receiver_position_beacons_temp_table_name} AS rpbt - GROUP BY rpbt.name - ) AS sq - WHERE rpbt.name = sq.name AND rpbt.timestamp = sq.timestamp - ON CONFLICT (name) DO UPDATE - SET - location = EXCLUDED.location, - altitude = EXCLUDED.altitude, - lastseen = EXCLUDED.lastseen, - timestamp = EXCLUDED.timestamp - """) - - # Update agl - cursor.execute(f""" - UPDATE {aircraft_position_beacons_temp_table_name} AS apbt - SET - agl = ST_Value(e.rast, apbt.location) - FROM elevation AS e - WHERE ST_Intersects(apbt.location, e.rast) - """) - - # ... update receiver related attributes: distance, radial, quality - cursor.execute(f""" - UPDATE {aircraft_position_beacons_temp_table_name} AS apbt - SET - distance = CAST(ST_DistanceSphere(r.location, apbt.location) AS REAL), - radial = CASE WHEN Degrees(ST_Azimuth(r.location, apbt.location)) >= 359.5 THEN 0 ELSE CAST(Degrees(ST_Azimuth(r.location, apbt.location)) AS INT) END, - quality = CASE WHEN ST_DistanceSphere(r.location, apbt.location) > 0 THEN CAST(apbt.signal_quality + 20.0 * LOG(ST_DistanceSphere(r.location, apbt.location) / 10000) AS REAL) ELSE NULL END - FROM receivers AS r - WHERE apbt.receiver_name = r.name - """) - - # Update devices - cursor.execute(f""" - INSERT INTO devices AS d (name, address, firstseen, lastseen, aircraft_type, stealth, software_version, hardware_version, real_address) - SELECT DISTINCT ON (apbt.name) - apbt.name, - apbt.address, - apbt.reference_timestamp AS firstseen, - apbt.reference_timestamp AS lastseen, - apbt.aircraft_type, - apbt.stealth, - apbt.software_version, - apbt.hardware_version, - apbt.real_address - FROM {aircraft_position_beacons_temp_table_name} AS apbt, - ( - SELECT - apbt.name, - MAX(timestamp) AS timestamp - FROM {aircraft_position_beacons_temp_table_name} AS apbt - GROUP BY apbt.name - ) AS sq - WHERE apbt.name = sq.name AND apbt.timestamp = sq.timestamp - ON CONFLICT (name) DO UPDATE - SET - lastseen = EXCLUDED.lastseen, - aircraft_type = EXCLUDED.aircraft_type, - stealth = EXCLUDED.stealth, - software_version = COALESCE(EXCLUDED.software_version, d.software_version), - hardware_version = COALESCE(EXCLUDED.hardware_version, d.hardware_version), - real_address = COALESCE(EXCLUDED.real_address, d.real_address); - """) - - # Insert all the beacons - cursor.execute(f""" - INSERT INTO aircraft_beacons - SELECT * FROM {aircraft_position_beacons_temp_table_name} - ON CONFLICT DO NOTHING; - """) - cursor.execute(f""" - INSERT INTO receiver_beacons - SELECT * FROM {receiver_position_beacons_temp_table_name} - ON CONFLICT DO NOTHING; - """) - connection.commit() - - cursor.close() - connection.close() - - self.aircraft_position_beacons_buffer = StringIO() - self.receiver_position_beacons_buffer = StringIO() - - def _flush_status_beacons(self): - connection = db.engine.raw_connection() - cursor = connection.cursor() - - self.aircraft_status_beacons_buffer.seek(0) - self.receiver_status_beacons_buffer.seek(0) - - aircraft_status_beacons_temp_table_name = f"aircraft_status_beacons_temp_{str(time.time()).replace('.', '_')}" - receiver_status_beacons_temp_table_name = f"receiver_status_beacons_temp_{str(time.time()).replace('.', '_')}" - - cursor.execute(f"CREATE TEMPORARY TABLE {aircraft_status_beacons_temp_table_name} (LIKE aircraft_beacons) ON COMMIT DROP;") - cursor.execute(f"CREATE TEMPORARY TABLE {receiver_status_beacons_temp_table_name} (LIKE receiver_beacons) ON COMMIT DROP;") - - #cursor.copy_from(file=self.aircraft_status_beacons_buffer, table="aircraft_status_beacons_temp", sep=",", columns=AIRCRAFT_STATUS_BEACON_FIELDS) - cursor.copy_from(file=self.receiver_status_beacons_buffer, table=receiver_status_beacons_temp_table_name, sep=",", columns=RECEIVER_STATUS_BEACON_FIELDS) - - # Update receivers - cursor.execute(f""" - INSERT INTO receivers AS r (name, timestamp, version, platform) - SELECT DISTINCT ON (rsbt.name) - rsbt.name, - rsbt.timestamp, - rsbt.version, - rsbt.platform - FROM {receiver_status_beacons_temp_table_name} AS rsbt, - ( - SELECT - rsbt.name, - MAX(timestamp) AS timestamp - FROM {receiver_status_beacons_temp_table_name} AS rsbt - GROUP BY rsbt.name - ) AS sq - WHERE rsbt.name = sq.name AND rsbt.timestamp = sq.timestamp - ON CONFLICT (name) DO UPDATE - SET - version = EXCLUDED.version, - platform = EXCLUDED.platform; - """) - - # Update receiver_beacons - cursor.execute(f""" - INSERT INTO receiver_beacons AS rb (name, dstcall, receiver_name, timestamp, version, platform, reference_timestamp) - SELECT DISTINCT ON (rsbt.name) - rsbt.name, - rsbt.dstcall, - rsbt.receiver_name, - rsbt.timestamp, - rsbt.version, - rsbt.platform, - rsbt.reference_timestamp - FROM {receiver_status_beacons_temp_table_name} AS rsbt, - ( - SELECT - rsbt.name, - MAX(timestamp) AS timestamp - FROM {receiver_status_beacons_temp_table_name} AS rsbt - GROUP BY rsbt.name - ) AS sq - WHERE rsbt.name = sq.name AND rsbt.timestamp = sq.timestamp - ON CONFLICT (name, receiver_name, timestamp) DO UPDATE - SET - version = EXCLUDED.version, - platform = EXCLUDED.platform; - """) - connection.commit() - - cursor.close() - connection.close() - - self.aircraft_status_beacons_buffer = StringIO() - self.receiver_status_beacons_buffer = StringIO() - - def flush(self): - self._flush_position_beacons() - self._flush_status_beacons() - -def convert(sourcefile): - print("Fast scan of file '{}'...".format(sourcefile), end='') - with open_file(sourcefile) as filehandler: - total_lines, reference_timestamp = initial_file_scan(filehandler) - print("done") - - if reference_timestamp is not None: - auto_update_timestamp = True - postfix = str(reference_timestamp.total_seconds()) - else: - auto_update_timestamp = False - match = re.match(r".*OGN_log\.txt_([0-9]{4}\-[0-9]{2}\-[0-9]{2})\.gz$", sourcefile) - if match: - reference_timestamp = datetime.strptime(match.group(1), "%Y-%m-%d") + timedelta(hours=12) - postfix = reference_timestamp.strftime("%Y_%m_%d") - else: - current_app.logger.error("No reference time information. Skipping file: {}".format(sourcefile)) - return - - with open_file(sourcefile) as fin: - with FileFeeder(postfix=postfix, reference_timestamp=reference_timestamp, auto_update_timestamp=auto_update_timestamp) as feeder: - pbar = tqdm(fin, total=total_lines) - for line in pbar: - pbar.set_description("Importing {}".format(sourcefile)) - feeder.add(raw_string=line) - - -def calculate(ab_filename, rb_filename, target_filename): - sql_string = (""" - DROP TABLE IF EXISTS tmp_ab; - DROP TABLE IF EXISTS tmp_rb; - - CREATE TABLE tmp_ab - AS - SELECT * - FROM aircraft_beacons - WITH NO DATA; - - CREATE TABLE tmp_rb - AS - SELECT * - FROM receiver_beacons - WITH NO DATA; - - COPY tmp_ab FROM PROGRAM 'gunzip -c {ab_filename}' CSV DELIMITER ',' HEADER; - COPY tmp_rb FROM PROGRAM 'gunzip -c {rb_filename}' CSV DELIMITER ',' HEADER; - - COPY ( - WITH sq AS ( - SELECT - 'SRID=4326;' || ST_AsText(ab.location) AS location, - ab.altitude, - ab.name, - ab.dstcall, - ab.relay, - ab.receiver_name, - ab.timestamp, - CASE WHEN ab.track = 360 THEN 0 ELSE ab.track END, - ab.ground_speed, - ab.address_type, - ab.aircraft_type, - ab.stealth, - ab.address, - ab.climb_rate, - ab.turn_rate, - ab.signal_quality, - ab.error_count, - ab.frequency_offset, - ab.gps_quality_horizontal, - ab.gps_quality_vertical, - ab.software_version, - ab.hardware_version, - ab.real_address, - ab.signal_power, - CAST(ST_DistanceSphere(rb.location, ab.location) AS REAL) AS distance, - CASE WHEN Degrees(ST_Azimuth(rb.location, ab.location)) >= 359.5 THEN 0 ELSE CAST(Degrees(ST_Azimuth(rb.location, ab.location)) AS INT) END AS radial, - CASE WHEN ST_DistanceSphere(rb.location, ab.location) > 0 THEN CAST(ab.signal_quality + 20.0 * LOG(ST_DistanceSphere(rb.location, ab.location) / 10000) AS REAL) ELSE NULL END quality, - ab.location_mgrs, - ab.location_mgrs_short, - ab.altitude - ST_Value(e.rast, ab.location) AS agl - FROM tmp_ab AS ab, elevation AS e, (SELECT name, MAX(location) AS location FROM tmp_rb GROUP BY name) AS rb - WHERE ab.receiver_name = rb.name AND ST_Intersects(ab.location, e.rast) - ) - - SELECT DISTINCT ON (name, receiver_name, timestamp) * - FROM sq - ) TO PROGRAM 'gzip > {target_filename}' CSV DELIMITER ',' HEADER; - - COPY ( - SELECT DISTINCT ON (name, receiver_name, timestamp) * - FROM tmp_rb AS rb - ) TO PROGRAM 'gzip > {rb_filename}2' CSV DELIMITER ',' HEADER; - """.format(ab_filename=ab_filename, rb_filename=rb_filename, target_filename=target_filename)) - - db.session.execute(sql_string) diff --git a/app/gateway/message_handling.py b/app/gateway/message_handling.py new file mode 100644 index 0000000..d240f12 --- /dev/null +++ b/app/gateway/message_handling.py @@ -0,0 +1,431 @@ +import os +import time +from io import StringIO + +from app import db +from app.model import AircraftType +from app.utils import get_sql_trustworthy + +basepath = os.path.dirname(os.path.realpath(__file__)) + +# define fields we want to proceed +SENDER_POSITION_BEACON_FIELDS = [ + "reference_timestamp", + + "name", + "dstcall", + "relay", + "receiver_name", + "timestamp", + "location", + + "track", + "ground_speed", + "altitude", + + "address_type", + "aircraft_type", + "stealth", + "address", + "climb_rate", + "turn_rate", + "signal_quality", + "error_count", + "frequency_offset", + "gps_quality_horizontal", + "gps_quality_vertical", + "software_version", + "hardware_version", + "real_address", + "signal_power", + + "distance", + "bearing", + "normalized_quality", + + "location_mgrs", + "location_mgrs_short", + "agl", +] + +RECEIVER_POSITION_BEACON_FIELDS = [ + "reference_timestamp", + + "name", + "dstcall", + "receiver_name", + "timestamp", + "location", + + "altitude", + + "location_mgrs", + "location_mgrs_short", + "agl", +] + +RECEIVER_STATUS_BEACON_FIELDS = [ + "reference_timestamp", + + "name", + "dstcall", + "receiver_name", + "timestamp", + + "version", + "platform", + + "cpu_temp", + "rec_input_noise", +] + + +def sender_position_message_to_csv_string(message, none_character=''): + """ + Convert sender_position_messages to csv string. + + :param dict message: dict of sender position messages from the parser + :param str none_character: '' for a file, '\\N' for Postgresql COPY + """ + + csv_string = "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16},{17},{18},{19},{20},{21},{22},{23},{24},{25},{26},{27},{28},{29},{30}\n".format( + message['reference_timestamp'], + + message['name'], + message['dstcall'], + message['relay'] if 'relay' in message and message['relay'] else none_character, + message['receiver_name'], + message['timestamp'], + message['location'], + + message['track'] if 'track' in message and message['track'] else none_character, + message['ground_speed'] if 'ground_speed' in message and message['ground_speed'] else none_character, + int(message['altitude']) if message['altitude'] else none_character, + + message['address_type'] if 'address_type' in message and message['address_type'] else none_character, #10 + message['aircraft_type'].name if 'aircraft_type' in message and message['aircraft_type'] else AircraftType.UNKNOWN.name, + message['stealth'] if 'stealth' in message and message['stealth'] else none_character, + message['address'] if 'address' in message and message['address'] else none_character, + message['climb_rate'] if 'climb_rate' in message and message['climb_rate'] else none_character, + message['turn_rate'] if 'turn_rate' in message and message['turn_rate'] else none_character, + message['signal_quality'] if 'signal_quality' in message and message['signal_quality'] else none_character, + message['error_count'] if 'error_count' in message and message['error_count'] else none_character, + message['frequency_offset'] if 'frequency_offset' in message and message['frequency_offset'] else none_character, + message['gps_quality_horizontal'] if 'gps_quality_horizontal' in message and message['gps_quality_horizontal'] else none_character, + message['gps_quality_vertical'] if 'gps_quality_vertical' in message and message['gps_quality_vertical'] else none_character, #20 + message['software_version'] if 'software_version' in message and message['software_version'] else none_character, + message['hardware_version'] if 'hardware_version' in message and message['hardware_version'] else none_character, + message['real_address'] if 'real_address' in message and message['real_address'] else none_character, + message['signal_power'] if 'signal_power' in message and message['signal_power'] else none_character, + + message['distance'] if 'distance' in message and message['distance'] else none_character, + message['bearing'] if 'bearing' in message and message['bearing'] else none_character, + message['normalized_quality'] if 'normalized_quality' in message and message['normalized_quality'] else none_character, + + message['location_mgrs'], + message['location_mgrs_short'], + message['agl'] if 'agl' in message else none_character, + ) + return csv_string + + +def receiver_position_message_to_csv_string(message, none_character=''): + csv_string = "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}\n".format( + message['reference_timestamp'], + + message['name'], + message['dstcall'], + message['receiver_name'], + message['timestamp'], + message['location'], + + int(message['altitude']) if message['altitude'] else none_character, + + message['location_mgrs'], + message['location_mgrs_short'], + message['agl'] if 'agl' in message else none_character, + ) + return csv_string + + +def receiver_status_message_to_csv_string(message, none_character=''): + csv_string = "{0},{1},{2},{3},{4},{5},{6},{7},{8}\n".format( + message['reference_timestamp'], + + message['name'], + message['dstcall'], + message['receiver_name'], + message['timestamp'], + + message['version'] if 'version' in message else none_character, + message['platform'] if 'platform' in message else none_character, + + message['cpu_temp'] if 'cpu_temp' in message else none_character, + message['rec_input_noise'] if 'rec_input_noise' in message else none_character, + + ) + return csv_string + + +def sender_position_csv_strings_to_db(lines): + timestamp_string = str(time.time()).replace('.', '_') + tmp_tablename = f'sender_positions_{timestamp_string}' + + connection = db.engine.raw_connection() + cursor = connection.cursor() + + string_buffer = StringIO() + string_buffer.writelines(lines) + string_buffer.seek(0) + + cursor.execute(f"CREATE TEMPORARY TABLE {tmp_tablename} (LIKE sender_positions) ON COMMIT DROP;") + cursor.copy_from(file=string_buffer, table=tmp_tablename, sep=",", columns=SENDER_POSITION_BEACON_FIELDS) + + # Update agl + cursor.execute(f""" + UPDATE {tmp_tablename} AS tmp + SET + agl = tmp.altitude - ST_Value(e.rast, tmp.location) + FROM elevation AS e + WHERE ST_Intersects(tmp.location, e.rast); + """) + + # Update sender position statistics + cursor.execute(f""" + INSERT INTO sender_position_statistics AS sps (date, dstcall, address_type, aircraft_type, stealth, software_version, hardware_version, messages_count) + SELECT + tmp.reference_timestamp::DATE AS date, + tmp.dstcall, + tmp.address_type, + tmp.aircraft_type, + tmp.stealth, + tmp.software_version, + tmp.hardware_version, + COUNT(tmp.*) AS messages_count + FROM {tmp_tablename} AS tmp + GROUP BY date, dstcall, address_type, aircraft_type, stealth, software_version, hardware_version + ON CONFLICT (date, dstcall, address_type, aircraft_type, stealth, software_version, hardware_version) DO UPDATE + SET + messages_count = EXCLUDED.messages_count + sps.messages_count; + """) + + # Update senders + cursor.execute(f""" + INSERT INTO senders AS s (firstseen, lastseen, name, aircraft_type, stealth, address, software_version, hardware_version, real_address) + SELECT DISTINCT ON (tmp.name) + tmp.reference_timestamp AS firstseen, + tmp.reference_timestamp AS lastseen, + + tmp.name, + + tmp.aircraft_type, + tmp.stealth, + tmp.address, + tmp.software_version, + tmp.hardware_version, + tmp.real_address + FROM {tmp_tablename} AS tmp + WHERE tmp.name NOT LIKE 'RND%' + ON CONFLICT (name) DO UPDATE + SET + lastseen = GREATEST(EXCLUDED.lastseen, s.lastseen), + aircraft_type = EXCLUDED.aircraft_type, + stealth = EXCLUDED.stealth, + address = EXCLUDED.address, + software_version = COALESCE(EXCLUDED.software_version, s.software_version), + hardware_version = COALESCE(EXCLUDED.hardware_version, s.hardware_version), + real_address = COALESCE(EXCLUDED.real_address, s.real_address); + """) + + # Update sender_infos FK -> senders + cursor.execute(f""" + UPDATE sender_infos AS si + SET sender_id = s.id + FROM senders AS s + WHERE si.sender_id IS NULL AND s.address = si.address; + """) + + SQL_TRUSTWORTHY = get_sql_trustworthy(source_table_alias='tmp') + + # Update coverage statistics + cursor.execute(f""" + INSERT INTO coverage_statistics AS rs (date, location_mgrs_short, sender_id, receiver_id, is_trustworthy, max_distance, max_normalized_quality, messages_count) + SELECT + tmp.reference_timestamp::DATE AS date, + tmp.location_mgrs_short, + tmp.sender_id, + tmp.receiver_id, + + ({SQL_TRUSTWORTHY}) AS is_trustworthy, + + MAX(tmp.distance) AS max_distance, + MAX(tmp.normalized_quality) AS max_normalized_quality, + COUNT(tmp.*) AS messages_count + FROM (SELECT x.*, s.id AS sender_id, r.id AS receiver_id FROM {tmp_tablename} AS x INNER JOIN senders AS s ON x.name = s.name INNER JOIN receivers AS r ON x.receiver_name = r.name) AS tmp + GROUP BY date, location_mgrs_short, sender_id, receiver_id, is_trustworthy + ON CONFLICT (date, location_mgrs_short, sender_id, receiver_id, is_trustworthy) DO UPDATE + SET + max_distance = GREATEST(EXCLUDED.max_distance, rs.max_distance), + max_normalized_quality = GREATEST(EXCLUDED.max_normalized_quality, rs.max_normalized_quality), + messages_count = EXCLUDED.messages_count + rs.messages_count; + """) + + # Insert all the beacons + all_fields = ', '.join(SENDER_POSITION_BEACON_FIELDS) + cursor.execute(f""" + INSERT INTO sender_positions ({all_fields}) + SELECT {all_fields} FROM {tmp_tablename}; + """) + + connection.commit() + + cursor.close() + connection.close() + + +def receiver_position_csv_strings_to_db(lines): + timestamp_string = str(time.time()).replace('.', '_') + tmp_tablename = f'receiver_positions_{timestamp_string}' + + connection = db.engine.raw_connection() + cursor = connection.cursor() + + string_buffer = StringIO() + string_buffer.writelines(lines) + string_buffer.seek(0) + + cursor.execute(f"CREATE TEMPORARY TABLE {tmp_tablename} (LIKE receiver_positions) ON COMMIT DROP;") + cursor.copy_from(file=string_buffer, table=tmp_tablename, sep=",", columns=RECEIVER_POSITION_BEACON_FIELDS) + + # Update agl + cursor.execute(f""" + UPDATE {tmp_tablename} AS tmp + SET + agl = tmp.altitude - ST_Value(e.rast, tmp.location) + FROM elevation AS e + WHERE ST_Intersects(tmp.location, e.rast); + """) + + # Update receivers + cursor.execute(f""" + INSERT INTO receivers AS r (firstseen, lastseen, name, timestamp, location, altitude, agl) + SELECT DISTINCT ON (tmp.name) + tmp.reference_timestamp AS firstseen, + tmp.reference_timestamp AS lastseen, + + tmp.name, + tmp.timestamp, + tmp.location, + + tmp.altitude, + + tmp.agl + FROM {tmp_tablename} AS tmp, + ( + SELECT + tmp.name, + MAX(timestamp) AS timestamp + FROM {tmp_tablename} AS tmp + GROUP BY tmp.name + ) AS sq + WHERE tmp.name = sq.name AND tmp.timestamp = sq.timestamp AND tmp.name NOT LIKE 'RND%' + ON CONFLICT (name) DO UPDATE + SET + lastseen = EXCLUDED.lastseen, + timestamp = EXCLUDED.timestamp, + location = EXCLUDED.location, + altitude = EXCLUDED.altitude, + + agl = EXCLUDED.agl; + """) + + # Update receiver country and nearest airport + cursor.execute(f""" + UPDATE receivers AS r + SET + country_id = c.gid, + airport_id = ( + SELECT id + FROM airports AS a + WHERE + ST_Contains(a.border, r.location) + AND a.style IN (2,4,5) + ORDER BY ST_DistanceSphere(a.location, r.location) + LIMIT 1 + ) + FROM countries AS c + WHERE r.country_id IS NULL AND ST_Within(r.location, c.geom); + """) + + # Insert all the beacons + all_fields = ', '.join(RECEIVER_POSITION_BEACON_FIELDS) + cursor.execute(f""" + INSERT INTO receiver_positions ({all_fields}) + SELECT {all_fields} FROM {tmp_tablename}; + """) + + connection.commit() + + cursor.close() + connection.close() + + +def receiver_status_csv_strings_to_db(lines): + timestamp_string = str(time.time()).replace('.', '_') + tmp_tablename = f'receiver_status_{timestamp_string}' + + connection = db.engine.raw_connection() + cursor = connection.cursor() + + string_buffer = StringIO() + string_buffer.writelines(lines) + string_buffer.seek(0) + + cursor.execute(f"CREATE TEMPORARY TABLE {tmp_tablename} (LIKE receiver_statuses) ON COMMIT DROP;") + cursor.copy_from(file=string_buffer, table=tmp_tablename, sep=",", columns=RECEIVER_STATUS_BEACON_FIELDS) + + # Update receivers + cursor.execute(f""" + INSERT INTO receivers AS r (firstseen, lastseen, name, timestamp, version, platform, cpu_temp, rec_input_noise) + SELECT DISTINCT ON (tmp.name) + tmp.reference_timestamp AS firstseen, + tmp.reference_timestamp AS lastseen, + + tmp.name, + tmp.timestamp, + + tmp.version, + tmp.platform, + + tmp.cpu_temp, + tmp.rec_input_noise + FROM {tmp_tablename} AS tmp, + ( + SELECT + tmp.name, + MAX(timestamp) AS timestamp + FROM {tmp_tablename} AS tmp + GROUP BY tmp.name + ) AS sq + WHERE tmp.name = sq.name AND tmp.timestamp = sq.timestamp + ON CONFLICT (name) DO UPDATE + SET + lastseen = EXCLUDED.lastseen, + timestamp = EXCLUDED.timestamp, + version = EXCLUDED.version, + platform = EXCLUDED.platform, + cpu_temp = EXCLUDED.cpu_temp, + rec_input_noise = EXCLUDED.rec_input_noise; + """) + + # Insert all the beacons + all_fields = ', '.join(RECEIVER_STATUS_BEACON_FIELDS) + cursor.execute(f""" + INSERT INTO receiver_statuses ({all_fields}) + SELECT {all_fields} FROM {tmp_tablename}; + """) + + connection.commit() + + cursor.close() + connection.close() diff --git a/app/gateway/process_tools.py b/app/gateway/process_tools.py index 9cc46f0..c8e565b 100644 --- a/app/gateway/process_tools.py +++ b/app/gateway/process_tools.py @@ -37,74 +37,10 @@ class Timer(object): print("[{}]".format(self.name)) print("Elapsed: {}".format(time.time() - self.tstart)) - -def drop_tables(postfix): - """Drop tables for log file import.""" - - db.session.execute(""" - DROP TABLE IF EXISTS "aircraft_beacons_{postfix}"; - DROP TABLE IF EXISTS "receiver_beacons_{postfix}"; - """.format(postfix=postfix)) - db.session.commit() - - -def create_tables(postfix): - """Create tables for log file import.""" - - drop_tables(postfix) - db.session.execute(""" - CREATE TABLE aircraft_beacons_{postfix} AS TABLE aircraft_beacons WITH NO DATA; - CREATE TABLE receiver_beacons_{postfix} AS TABLE receiver_beacons WITH NO DATA; - """.format(postfix=postfix)) - db.session.commit() - - -def update_aircraft_beacons_bigdata(postfix): - """Calculates distance/radial and quality and computes the altitude above ground level. - Due to performance reasons we use a new table instead of updating the old.""" - - db.session.execute(""" - SELECT - ab.location, ab.altitude, ab.name, ab.dstcall, ab.relay, ab.receiver_name, ab.timestamp, ab.track, ab.ground_speed, - - ab.address_type, ab.aircraft_type, ab.stealth, ab.address, ab.climb_rate, ab.turn_rate, ab.signal_quality, ab.error_count, - ab.frequency_offset, ab.gps_quality_horizontal, ab.gps_quality_vertical, ab.software_version, ab.hardware_version, ab.real_address, ab.signal_power, - - ab.location_mgrs, - ab.location_mgrs_short, - - CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(ST_DistanceSphere(ab.location, r.location) AS REAL) ELSE NULL END AS distance, - CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL THEN CAST(degrees(ST_Azimuth(ab.location, r.location)) AS SMALLINT) % 360 ELSE NULL END AS radial, - CASE WHEN ab.location IS NOT NULL AND r.location IS NOT NULL AND ST_DistanceSphere(ab.location, r.location) > 0 AND ab.signal_quality IS NOT NULL - THEN CAST(signal_quality + 20*log(ST_DistanceSphere(ab.location, r.location)/10000) AS REAL) - ELSE NULL - END AS quality, - CAST((ab.altitude - subtable.elev_m) AS REAL) AS agl - INTO aircraft_beacons_{postfix}_temp - FROM - aircraft_beacons_{postfix} AS ab - JOIN LATERAL ( - SELECT ab.location, MAX(ST_NearestValue(e.rast, ab.location)) as elev_m - FROM elevation e - WHERE ST_Intersects(ab.location, e.rast) - GROUP BY ab.location - ) AS subtable ON TRUE, - (SELECT name, last(location, timestamp) AS location FROM receiver_beacons_{postfix} GROUP BY name) AS r - WHERE ab.receiver_name = r.name; - - DROP TABLE IF EXISTS "aircraft_beacons_{postfix}"; - ALTER TABLE "aircraft_beacons_{postfix}_temp" RENAME TO "aircraft_beacons_{postfix}"; - """.format(postfix=postfix)) - - -def export_to_path(postfix, path): +def export_to_path(path): connection = db.engine.raw_connection() cursor = connection.cursor() - aircraft_beacons_file = os.path.join(path, "aircraft_beacons_{postfix}.csv.gz".format(postfix=postfix)) + aircraft_beacons_file = os.path.join(path, "sender_positions.csv.gz") with gzip.open(aircraft_beacons_file, "wt", encoding="utf-8") as gzip_file: - cursor.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format("SELECT * FROM aircraft_beacons_{postfix}".format(postfix=postfix)), gzip_file) - - receiver_beacons_file = os.path.join(path, "receiver_beacons_{postfix}.csv.gz".format(postfix=postfix)) - with gzip.open(receiver_beacons_file, "wt") as gzip_file: - cursor.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format("SELECT * FROM receiver_beacons_{postfix}".format(postfix=postfix)), gzip_file) + cursor.copy_expert("COPY ({}) TO STDOUT WITH (DELIMITER ',', FORMAT CSV, HEADER, ENCODING 'UTF-8');".format("SELECT * FROM sender_positions"), gzip_file) diff --git a/app/main/__init__.py b/app/main/__init__.py index 7e35f16..fe53f94 100644 --- a/app/main/__init__.py +++ b/app/main/__init__.py @@ -3,3 +3,4 @@ from flask import Blueprint bp = Blueprint("main", __name__) import app.main.routes +import app.main.jinja_filters \ No newline at end of file diff --git a/app/main/jinja_filters.py b/app/main/jinja_filters.py new file mode 100644 index 0000000..615ac81 --- /dev/null +++ b/app/main/jinja_filters.py @@ -0,0 +1,61 @@ +from app.main import bp +from app.model import Airport, Sender, Receiver + +from flask import url_for +import time +import datetime +import math + +@bp.app_template_filter() +def timestamp_to_status(timestamp): + if datetime.datetime.utcnow() - timestamp < datetime.timedelta(minutes=10): + return 'OK' + elif datetime.datetime.utcnow() - timestamp < datetime.timedelta(hours=1): + return '?' + else: + return 'OFFLINE' + +@bp.app_template_filter() +def to_html_link(obj): + if isinstance(obj, Airport): + airport = obj + return f"""{airport.name}""" + + elif isinstance(obj, Sender): + sender = obj + if len(sender.infos) > 0 and len(sender.infos[0].registration) > 0: + return f"""{sender.infos[0].registration}""" + elif sender.address: + return f"""[{sender.address}]""" + else: + return f"""[{sender.name}]""" + + elif isinstance(obj, Receiver): + receiver = obj + return f"""{receiver.name}""" + + elif obj is None: + return "-" + + else: + raise NotImplementedError("cant apply filter 'to_html_link' to object {type(obj)}") + +@bp.app_template_filter() +def to_ordinal(rad): + deg = math.degrees(rad) + if deg >= 337.5 or deg < 22.5: + return "N" + elif deg >= 22.5 and deg < 67.5: + return "NW" + elif deg >= 67.5 and deg < 112.5: + return "W" + elif deg >= 112.5 and deg < 157.5: + return "SW" + elif deg >= 157.5 and deg < 202.5: + return "S" + elif deg >= 202.5 and deg < 247.5: + return "SE" + elif deg >= 247.5 and deg < 292.5: + return "E" + elif deg >= 292.5 and deg < 337.5: + return "NE" \ No newline at end of file diff --git a/app/main/live_routes.py b/app/main/live_routes.py deleted file mode 100644 index 9c6a621..0000000 --- a/app/main/live_routes.py +++ /dev/null @@ -1,87 +0,0 @@ -from flask import request, render_template, current_app -from flask_cors import cross_origin - -from app.backend.liveglidernet import rec, lxml -from app.main import bp - - -@bp.route("/live.html") -@cross_origin() -def live(): - return render_template("ogn_live.html", host=request.host) - - -@bp.route("/rec.php") -def rec_php(): - a = request.args.get("a") - z = request.args.get("z") - - xml = rec() - resp = current_app.make_response(xml) - resp.mimetype = "text/xml" - return resp - - -@bp.route("/lxml.php") -def lxml_php(): - a = request.args.get("a") - b = request.args.get("b") - c = request.args.get("c") - d = request.args.get("d") - e = request.args.get("e") - z = request.args.get("z") - - xml = lxml() - resp = current_app.make_response(xml) - resp.mimetype = "text/xml" - return resp - - -@bp.route("/pict/") -def pict(filename): - return current_app.send_static_file("ognlive/pict/" + filename) - - -@bp.route("/favicon.gif") -def favicon_gif(): - return current_app.send_static_file("ognlive/pict/favicon.gif") - - -@bp.route("/horizZoomControl.js") -def horizZoomControl_js(): - return current_app.send_static_file("ognlive/horizZoomControl.js") - - -@bp.route("/barogram.js") -def barogram_js(): - return current_app.send_static_file("ognlive/barogram.js") - - -@bp.route("/util.js") -def util_js(): - return current_app.send_static_file("ognlive/util.js") - - -@bp.route("/ogn.js") -def ogn_js(): - return current_app.send_static_file("ognlive/ogn.js") - - -@bp.route("/ol.js") -def ol_js(): - return current_app.send_static_file("ognlive/ol.js") - - -@bp.route("/osm.js") -def osm_js(): - return current_app.send_static_file("ognlive/osm.js") - - -@bp.route("/ol.css") -def ol_css(): - return current_app.send_static_file("ognlive/ol.css") - - -@bp.route("/osm.css") -def osm_css(): - return current_app.send_static_file("ognlive/osm.css") diff --git a/app/main/matplotlib_service.py b/app/main/matplotlib_service.py new file mode 100644 index 0000000..fad6064 --- /dev/null +++ b/app/main/matplotlib_service.py @@ -0,0 +1,42 @@ +from app import db +from app.model import * +import random +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.figure import Figure + +def create_range_figure2(sender_id): + fig = Figure() + axis = fig.add_subplot(1, 1, 1) + xs = range(100) + ys = [random.randint(1, 50) for x in xs] + axis.plot(xs, ys) + + return fig + +def create_range_figure(sender_id): + sds = db.session.query(SenderDirectionStatistic) \ + .filter(SenderDirectionStatistic.sender_id == sender_id) \ + .order_by(SenderDirectionStatistic.directions_count.desc()) \ + .limit(1) \ + .one() + + fig = Figure() + + direction_data = sds.direction_data + max_range = max([r['max_range']/1000.0 for r in direction_data]) + + theta = np.array([i['direction']/180*np.pi for i in direction_data]) + radii = np.array([i['max_range']/1000 if i['max_range'] > 0 else 0 for i in direction_data]) + width = np.array([13/180*np.pi for i in direction_data]) + colors = plt.cm.viridis(radii / max_range) + + ax = fig.add_subplot(111, projection='polar') + ax.bar(theta, radii, width=width, bottom=0.0, color=colors, edgecolor='b', alpha=0.5) + #ax.set_rticks([0, 25, 50, 75, 100, 125, 150]) + ax.set_theta_zero_location("N") + ax.set_theta_direction(-1) + + fig.suptitle(f"Range between sender '{sds.sender.name}' and receiver '{sds.receiver.name}'") + + return fig \ No newline at end of file diff --git a/app/main/routes.py b/app/main/routes.py index 42f7925..82074c3 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -1,12 +1,13 @@ -import datetime +from datetime import date, time, datetime from flask import request, render_template, send_file from app import db from app import cache -from app.model import Airport, Country, Device, Logbook, Receiver +from app.model import Airport, Country, Sender, SenderInfo, TakeoffLanding, Logbook, Receiver, SenderPosition, RelationStatistic, ReceiverStatistic, SenderStatistic from app.main import bp +from app.main.matplotlib_service import create_range_figure @cache.cached(key_prefix="countries_in_receivers") @@ -17,27 +18,25 @@ def get_countries_in_receivers(): @cache.cached(key_prefix="countries_in_logbook") -def get_countries_in_logbook(): +def get_used_countries(): query = db.session.query(Country.iso2).filter(Country.iso2 == Airport.country_code).filter(Logbook.takeoff_airport_id == Airport.id).order_by(Country.iso2).distinct(Country.iso2) - return [{"iso2": country[0]} for country in query.all()] @cache.memoize() -def get_airports_in_country(sel_country): - query = db.session.query(Airport.id, Airport.name).filter(Airport.country_code == sel_country).filter(Logbook.takeoff_airport_id == Airport.id).order_by(Airport.name).distinct(Airport.name) - - return [{"id": airport[0], "name": airport[1]} for airport in query.all()] +def get_used_airports_by_country(sel_country): + query = db.session.query(Airport).filter(Airport.country_code == sel_country).filter(Logbook.takeoff_airport_id == Airport.id).order_by(Airport.name).distinct(Airport.name) + return [used_airport for used_airport in query] @cache.memoize() def get_dates_for_airport(sel_airport): query = ( - db.session.query(db.func.date(Logbook.reftime), db.func.count(Logbook.id).label("logbook_count")) + db.session.query(db.func.date(Logbook.reference), db.func.count(Logbook.id).label("logbook_count")) .filter(Airport.id == sel_airport) .filter(db.or_(Airport.id == Logbook.takeoff_airport_id, Airport.id == Logbook.landing_airport_id)) - .group_by(db.func.date(Logbook.reftime)) - .order_by(db.func.date(Logbook.reftime).desc()) + .group_by(db.func.date(Logbook.reference)) + .order_by(db.func.date(Logbook.reference).desc()) ) return [{"date": date, "logbook_count": logbook_count} for (date, logbook_count) in query.all()] @@ -46,21 +45,54 @@ def get_dates_for_airport(sel_airport): @bp.route("/") @bp.route("/index.html") def index(): - return render_template("base.html") + today_beginning = datetime.combine(date.today(), time()) + + senders_today = db.session.query(db.func.count(Sender.id)).filter(Sender.lastseen>=today_beginning).one()[0] + receivers_today = db.session.query(db.func.count(Receiver.id)).filter(Receiver.lastseen>=today_beginning).one()[0] + takeoffs_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp>=today_beginning, TakeoffLanding.is_takeoff==True)).one()[0] + landings_today = db.session.query(db.func.count(TakeoffLanding.id)).filter(db.and_(TakeoffLanding.timestamp>=today_beginning, TakeoffLanding.is_takeoff==False)).one()[0] + sender_positions_today = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).filter(ReceiverStatistic.date==date.today()).one()[0] + sender_positions_total = db.session.query(db.func.sum(ReceiverStatistic.messages_count)).one()[0] + + last_logbook_entries = db.session.query(Logbook).order_by(Logbook.reference.desc()).limit(10) + return render_template("index.html", + senders_today=senders_today, + receivers_today=receivers_today, + takeoffs_today=takeoffs_today, + landings_today=landings_today, + sender_positions_today=sender_positions_today, + sender_positions_total=sender_positions_total, + logbook=last_logbook_entries) -@bp.route("/devices.html", methods=["GET", "POST"]) -def devices(): - devices = db.session.query(Device).order_by(Device.address) - return render_template("devices.html", devices=devices) +@bp.route("/senders.html", methods=["GET", "POST"]) +def senders(): + senders = db.session.query(Sender) \ + .options(db.joinedload(Sender.infos)) \ + .order_by(Sender.name) + return render_template("senders.html", senders=senders) -@bp.route("/device_detail.html", methods=["GET", "POST"]) -def device_detail(): - device_name = request.args.get("device_name") - device = db.session.query(Device).filter(Device.name == device_name).one() +@bp.route("/sender_detail.html", methods=["GET", "POST"]) +def sender_detail(): + sender_id = request.args.get("sender_id") + sender = db.session.query(Sender).filter(Sender.id == sender_id).one() - return render_template("device_detail.html", title="Device", device=device) + return render_template("sender_detail.html", title="Sender", sender=sender) + +@bp.route("/range_view.png") +def range_view(): + import io + from flask import Response + + from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas + + sender_id = request.args.get("sender_id") + + fig = create_range_figure(sender_id) + output = io.BytesIO() + FigureCanvas(fig).print_png(output) + return Response(output.getvalue(), mimetype='image/png') @bp.route("/receivers.html") @@ -71,41 +103,34 @@ def receivers(): # Get receiver selection list if sel_country: - receivers = db.session.query(Receiver).filter(db.and_(Receiver.country_id == Country.gid, Country.iso2 == sel_country)).order_by(Receiver.name) + receivers = db.session.query(Receiver) \ + .options(db.joinedload(Receiver.airport)) \ + .filter(db.and_(Receiver.country_id == Country.gid, Country.iso2 == sel_country)) \ + .order_by(Receiver.name) else: - receivers = db.session.query(Receiver).order_by(Receiver.name) + receivers = db.session.query(Receiver) \ + .options(db.joinedload(Receiver.airport)) \ + .order_by(Receiver.name) return render_template("receivers.html", title="Receivers", sel_country=sel_country, countries=countries, receivers=receivers) @bp.route("/receiver_detail.html") def receiver_detail(): - receiver_name = request.args.get("receiver_name") + receiver_id = request.args.get("receiver_id") - receiver = db.session.query(Receiver).filter(Receiver.name == receiver_name).one() - - airport = ( - db.session.query(Airport) - .filter( - db.and_( - Receiver.name == receiver_name, - db.func.st_contains(db.func.st_buffer(Receiver.location_wkt, 0.5), Airport.location_wkt), - db.func.st_distance_sphere(Airport.location_wkt, Receiver.location_wkt) < 1000, - ) - ) - .filter(Airport.style.in_((2, 4, 5))) - ) - return render_template("receiver_detail.html", title="Receiver Detail", receiver=receiver, airport=airport.first()) + receiver = db.session.query(Receiver).filter(Receiver.id == receiver_id).one() + return render_template("receiver_detail.html", title="Receiver Detail", receiver=receiver) @bp.route("/airports.html", methods=["GET", "POST"]) def airports(): sel_country = request.args.get("country") - countries = get_countries_in_logbook() + countries = get_used_countries() if sel_country: - airports = get_airports_in_country(sel_country) + airports = get_used_airports_by_country(sel_country) else: airports = [] @@ -116,41 +141,41 @@ def airports(): @bp.route("/airport_detail.html") def airport_detail(): - sel_airport = request.args.get("airport") + sel_airport = request.args.get("airport_id") airport = db.session.query(Airport).filter(Airport.id == sel_airport) - devices = db.session.query(Device).join(Logbook).filter(Logbook.takeoff_airport_id == sel_airport).order_by(Device.address) + senders = db.session.query(Sender).join(Logbook).filter(Logbook.takeoff_airport_id == sel_airport).order_by(Sender.name) - return render_template("airport_detail.html", title="Airport Detail", airport=airport.one(), devices=devices) + return render_template("airport_detail.html", title="Airport Detail", airport=airport.one(), senders=senders) @bp.route("/logbook.html", methods=["GET", "POST"]) def logbook(): sel_country = request.args.get("country") - sel_airport = request.args.get("airport") + sel_airport_id = request.args.get("airport_id") sel_date = request.args.get("date") - sel_device_id = request.args.get("device_id") + sel_sender_id = request.args.get("sender_id") - countries = get_countries_in_logbook() + countries = get_used_countries() if sel_country: - airports = get_airports_in_country(sel_country) + airports = get_used_airports_by_country(sel_country) else: airports = [] - if sel_airport: - sel_airport = int(sel_airport) - if sel_airport not in [airport["id"] for airport in airports]: - sel_airport = None + if sel_airport_id: + sel_airport_id = int(sel_airport_id) + if sel_airport_id not in [airport.id for airport in airports]: + sel_airport_id = None sel_date = None - dates = get_dates_for_airport(sel_airport) + dates = get_dates_for_airport(sel_airport_id) else: dates = [] if sel_date: - sel_date = datetime.datetime.strptime(sel_date, "%Y-%m-%d").date() + sel_date = datetime.strptime(sel_date, "%Y-%m-%d").date() if sel_date not in [entry["date"] for entry in dates]: sel_date = dates[0]["date"] elif len(dates) > 0: @@ -158,21 +183,21 @@ def logbook(): # Get Logbook filters = [] - if sel_airport: - filters.append(db.or_(Logbook.takeoff_airport_id == sel_airport, Logbook.landing_airport_id == sel_airport)) + if sel_airport_id: + filters.append(db.or_(Logbook.takeoff_airport_id == sel_airport_id, Logbook.landing_airport_id == sel_airport_id)) if sel_date: - filters.append(db.func.date(Logbook.reftime) == sel_date) + filters.append(db.func.date(Logbook.reference) == sel_date) - if sel_device_id: - filters.append(Logbook.device_id == sel_device_id) + if sel_sender_id: + filters.append(Logbook.sender_id == sel_sender_id) if len(filters) > 0: - logbook = db.session.query(Logbook).filter(*filters).order_by(Logbook.reftime) + logbook = db.session.query(Logbook).filter(*filters).order_by(Logbook.reference) else: logbook = None - return render_template("logbook.html", title="Logbook", sel_country=sel_country, countries=countries, sel_airport=sel_airport, airports=airports, sel_date=sel_date, dates=dates, logbook=logbook) + return render_template("logbook.html", title="Logbook", sel_country=sel_country, countries=countries, sel_airport_id=sel_airport_id, airports=airports, sel_date=sel_date, dates=dates, logbook=logbook) @bp.route("/download.html") @@ -184,3 +209,25 @@ def download_flight(): buffer.seek(0) return send_file(buffer, as_attachment=True, attachment_filename="wtf.igc", mimetype="text/plain") + +@bp.route("/sender_ranking.html") +def sender_ranking(): + sender_statistics = db.session.query(SenderStatistic) \ + .filter(db.and_(SenderStatistic.date==date.today(), SenderStatistic.is_trustworthy==True)) \ + .order_by(SenderStatistic.max_distance.desc()) \ + .all() + + return render_template("sender_ranking.html", + title="Sender Ranking", + ranking=sender_statistics) + +@bp.route("/receiver_ranking.html") +def receiver_ranking(): + receiver_statistics = db.session.query(ReceiverStatistic) \ + .filter(db.and_(ReceiverStatistic.date==date.today(), ReceiverStatistic.is_trustworthy==True)) \ + .order_by(ReceiverStatistic.max_distance.desc()) \ + .all() + + return render_template("receiver_ranking.html", + title="Receiver Ranking", + ranking=receiver_statistics) diff --git a/app/model/__init__.py b/app/model/__init__.py index 2942398..89fe239 100644 --- a/app/model/__init__.py +++ b/app/model/__init__.py @@ -1,15 +1,22 @@ # flake8: noqa from .aircraft_type import AircraftType -from .beacon import Beacon from .country import Country -from .device import Device -from .device_info import DeviceInfo -from .device_info_origin import DeviceInfoOrigin -from .aircraft_beacon import AircraftBeacon -from .receiver_beacon import ReceiverBeacon +from .sender import Sender +from .sender_info_origin import SenderInfoOrigin +from .sender_info import SenderInfo +from .sender_position import SenderPosition +from .receiver_position import ReceiverPosition +from .receiver_status import ReceiverStatus from .receiver import Receiver from .takeoff_landing import TakeoffLanding from .airport import Airport from .logbook import Logbook from .geo import Location + +from .relation_statistic import RelationStatistic +from .coverage_statistic import CoverageStatistic +from .sender_statistic import SenderStatistic +from .receiver_statistic import ReceiverStatistic +from .sender_position_statistic import SenderPositionStatistic +from .sender_direction_statistic import SenderDirectionStatistic diff --git a/app/model/aircraft_beacon.py b/app/model/aircraft_beacon.py deleted file mode 100644 index c6925ee..0000000 --- a/app/model/aircraft_beacon.py +++ /dev/null @@ -1,60 +0,0 @@ -from sqlalchemy.sql import func -from app import db - -from .beacon import Beacon -from .aircraft_type import AircraftType - - -class AircraftBeacon(Beacon): - __tablename__ = "aircraft_beacons" - - # Flarm specific data - address_type = db.Column(db.SmallInteger) - aircraft_type = db.Column(db.Enum(AircraftType), nullable=False, default=AircraftType.UNKNOWN) - stealth = db.Column(db.Boolean) - address = db.Column(db.String) - climb_rate = db.Column(db.Float(precision=2)) - turn_rate = db.Column(db.Float(precision=2)) - signal_quality = db.Column(db.Float(precision=2)) - error_count = db.Column(db.SmallInteger) - frequency_offset = db.Column(db.Float(precision=2)) - gps_quality_horizontal = db.Column(db.SmallInteger) - gps_quality_vertical = db.Column(db.SmallInteger) - software_version = db.Column(db.Float(precision=2)) - hardware_version = db.Column(db.SmallInteger) - real_address = db.Column(db.String(6)) - signal_power = db.Column(db.Float(precision=2)) - proximity = None - - # Calculated values - distance = db.Column(db.Float(precision=2)) - radial = db.Column(db.SmallInteger) - quality = db.Column(db.Float(precision=2)) # signal quality normalized to 10km - location_mgrs = db.Column(db.String(15)) # full mgrs (15 chars) - location_mgrs_short = db.Column(db.String(9)) # reduced mgrs (9 chars), e.g. used for melissas range tool - agl = db.Column(db.Float(precision=2)) - - def __repr__(self): - return "" % ( - self.address_type, - self.aircraft_type, - self.stealth, - self.address, - self.climb_rate, - self.turn_rate, - self.signal_quality, - self.error_count, - self.frequency_offset, - self.gps_quality_horizontal, - self.gps_quality_vertical, - self.software_version, - self.hardware_version, - self.real_address, - self.signal_power, - self.distance, - self.radial, - self.quality, - self.location_mgrs, - self.location_mgrs_short, - self.agl, - ) diff --git a/app/model/beacon.py b/app/model/beacon.py deleted file mode 100644 index 9bb4ab6..0000000 --- a/app/model/beacon.py +++ /dev/null @@ -1,33 +0,0 @@ -from geoalchemy2.shape import to_shape -from geoalchemy2.types import Geometry -from sqlalchemy.ext.declarative import AbstractConcreteBase -from sqlalchemy.ext.hybrid import hybrid_property - -from .geo import Location - -from app import db - - -class Beacon(AbstractConcreteBase, db.Model): - # APRS data - location = db.Column("location", Geometry("POINT", srid=4326)) - altitude = db.Column(db.Float(precision=2)) - - name = db.Column(db.String, primary_key=True) - dstcall = db.Column(db.String) - relay = db.Column(db.String) - receiver_name = db.Column(db.String(9), primary_key=True) - timestamp = db.Column(db.DateTime, primary_key=True) - symboltable = None - symbolcode = None - track = db.Column(db.SmallInteger) - ground_speed = db.Column(db.Float(precision=2)) - comment = None - - # Type information - beacon_type = None - aprs_type = None - - # Debug information - raw_message = None - reference_timestamp = db.Column(db.DateTime, nullable=False) diff --git a/app/model/coverage_statistic.py b/app/model/coverage_statistic.py new file mode 100644 index 0000000..5afc968 --- /dev/null +++ b/app/model/coverage_statistic.py @@ -0,0 +1,27 @@ +from app import db + +from sqlalchemy import Index +from sqlalchemy.orm import backref + + +class CoverageStatistic(db.Model): + __tablename__ = "coverage_statistics" + + id = db.Column(db.Integer, primary_key=True) + + date = db.Column(db.Date) + location_mgrs_short = db.Column(db.String(9)) + is_trustworthy = db.Column(db.Boolean) + + messages_count = db.Column(db.Integer) + max_distance = db.Column(db.Float(precision=2)) + max_normalized_quality = db.Column(db.Float(precision=2)) + + # Relations + sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("coverage_stats", order_by=date)) + + receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True) + receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("coverage_stats", order_by=date)) + + __table_args__ = (Index('idx_coverage_statistics_uc', 'date', 'location_mgrs_short', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), ) diff --git a/app/model/device_info_origin.py b/app/model/device_info_origin.py deleted file mode 100644 index a779697..0000000 --- a/app/model/device_info_origin.py +++ /dev/null @@ -1,8 +0,0 @@ -import enum - - -class DeviceInfoOrigin(enum.Enum): - UNKNOWN = 0 - OGN_DDB = 1 - FLARMNET = 2 - USER_DEFINED = 3 diff --git a/app/model/logbook.py b/app/model/logbook.py index 17a6aa9..94dc224 100644 --- a/app/model/logbook.py +++ b/app/model/logbook.py @@ -1,7 +1,8 @@ from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.sql import null, case +from sqlalchemy.orm import backref from app import db -from app.model import Device +from app.model import Sender class Logbook(db.Model): @@ -9,8 +10,6 @@ class Logbook(db.Model): id = db.Column(db.Integer, primary_key=True) - reftime = db.Column(db.DateTime, index=True) - address = db.Column(db.String, index=True) takeoff_timestamp = db.Column(db.DateTime) takeoff_track = db.Column(db.SmallInteger) landing_timestamp = db.Column(db.DateTime) @@ -18,15 +17,16 @@ class Logbook(db.Model): max_altitude = db.Column(db.Float(precision=2)) # Relations + sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) + #sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("logbook_entries", order_by=reference.desc()) # TODO: does not work... + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("logbook_entries", order_by=case({True: takeoff_timestamp, False: landing_timestamp}, takeoff_timestamp != null()).desc())) + takeoff_airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True) takeoff_airport = db.relationship("Airport", foreign_keys=[takeoff_airport_id]) landing_airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True) landing_airport = db.relationship("Airport", foreign_keys=[landing_airport_id]) - def get_device(self): - return db.session.query(Device).filter(Device.address == self.address).one() - @hybrid_property def duration(self): return None if (self.landing_timestamp is None or self.takeoff_timestamp is None) else self.landing_timestamp - self.takeoff_timestamp @@ -34,3 +34,11 @@ class Logbook(db.Model): @duration.expression def duration(cls): return case({False: None, True: cls.landing_timestamp - cls.takeoff_timestamp}, cls.landing_timestamp != null() and cls.takeoff_timestamp != null()) + + @hybrid_property + def reference(self): + return self.takeoff_timestamp if self.takeoff_timestamp is not None else self.landing_timestamp + + @reference.expression + def reference(cls): + return case({True: cls.takeoff_timestamp, False: cls.landing_timestamp}, cls.takeoff_timestamp != null()) diff --git a/app/model/receiver.py b/app/model/receiver.py index 8b5f465..74b1b2b 100644 --- a/app/model/receiver.py +++ b/app/model/receiver.py @@ -4,12 +4,17 @@ from geoalchemy2.types import Geometry from .geo import Location from app import db +from sqlalchemy import Index + +from .airport import Airport class Receiver(db.Model): __tablename__ = "receivers" - name = db.Column(db.String(9), primary_key=True) + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(9)) + location_wkt = db.Column("location", Geometry("POINT", srid=4326)) altitude = db.Column(db.Float(precision=2)) @@ -18,11 +23,20 @@ class Receiver(db.Model): timestamp = db.Column(db.DateTime, index=True) version = db.Column(db.String) platform = db.Column(db.String) + cpu_temp = db.Column(db.Float(precision=2)) + rec_input_noise = db.Column(db.Float(precision=2)) + + agl = db.Column(db.Float(precision=2)) # Relations country_id = db.Column(db.Integer, db.ForeignKey("countries.gid", ondelete="SET NULL"), index=True) country = db.relationship("Country", foreign_keys=[country_id], backref=db.backref("receivers", order_by="Receiver.name.asc()")) + airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="CASCADE"), index=True) + airport = db.relationship("Airport", foreign_keys=[airport_id], backref=db.backref("receivers", order_by="Receiver.name.asc()")) + + __table_args__ = (Index('idx_receivers_name_uc', 'name', unique=True), ) + @property def location(self): if self.location_wkt is None: @@ -30,3 +44,15 @@ class Receiver(db.Model): coords = to_shape(self.location_wkt) return Location(lat=coords.y, lon=coords.x) + + def airports_nearby(self): + query = ( + db.session.query(Airport, db.func.st_distance_sphere(self.location_wkt, Airport.location_wkt), db.func.st_azimuth(self.location_wkt, Airport.location_wkt)) + .filter(db.func.st_contains(db.func.st_buffer(Airport.location_wkt, 1), self.location_wkt)) + .filter(Airport.style.in_((2,4,5))) + .order_by(db.func.st_distance_sphere(self.location_wkt, Airport.location_wkt).asc()) + .limit(5) + ) + airports = [(airport,distance,azimuth) for airport, distance, azimuth in query] + return airports + diff --git a/app/model/receiver_beacon.py b/app/model/receiver_beacon.py deleted file mode 100644 index 4374e82..0000000 --- a/app/model/receiver_beacon.py +++ /dev/null @@ -1,28 +0,0 @@ -from app import db - -from .beacon import Beacon - - -class ReceiverBeacon(Beacon): - __tablename__ = "receiver_beacons" - - # disable irrelevant aprs fields - relay = None - track = None - ground_speed = None - - # Receiver specific data - version = db.Column(db.String) - platform = db.Column(db.String) - - def __repr__(self): - return "" % ( - self.name, - self.location, - self.altitude, - self.dstcall, - self.receiver_name, - self.timestamp, - self.version, - self.platform - ) diff --git a/app/model/receiver_position.py b/app/model/receiver_position.py new file mode 100644 index 0000000..401ceea --- /dev/null +++ b/app/model/receiver_position.py @@ -0,0 +1,39 @@ +from geoalchemy2.types import Geometry +from app import db + + +class ReceiverPosition(db.Model): + __tablename__ = "receiver_positions" + + reference_timestamp = db.Column(db.DateTime, primary_key=True) + + # APRS data + name = db.Column(db.String) + dstcall = db.Column(db.String) + #relay = db.Column(db.String) + receiver_name = db.Column(db.String(9)) + timestamp = db.Column(db.DateTime) + location = db.Column("location", Geometry("POINT", srid=4326)) + symboltable = None + symbolcode = None + + #track = db.Column(db.SmallInteger) + #ground_speed = db.Column(db.Float(precision=2)) + altitude = db.Column(db.Float(precision=2)) + + comment = None + + # Type information + beacon_type = None + aprs_type = None + + # Debug information + raw_message = None + + # Receiver specific data + user_comment = None + + # Calculated values (from this software) + location_mgrs = db.Column(db.String(15)) # full mgrs (15 chars) + location_mgrs_short = db.Column(db.String(9)) # reduced mgrs (9 chars), e.g. used for melissas range tool + agl = db.Column(db.Float(precision=2)) diff --git a/app/model/receiver_statistic.py b/app/model/receiver_statistic.py new file mode 100644 index 0000000..5bba591 --- /dev/null +++ b/app/model/receiver_statistic.py @@ -0,0 +1,25 @@ +from app import db + +from sqlalchemy import Index +from sqlalchemy.orm import backref + + +class ReceiverStatistic(db.Model): + __tablename__ = "receiver_statistics" + + id = db.Column(db.Integer, primary_key=True) + + date = db.Column(db.Date) + is_trustworthy = db.Column(db.Boolean) + + max_distance = db.Column(db.Float(precision=2)) + max_normalized_quality = db.Column(db.Float(precision=2)) + messages_count = db.Column(db.Integer) + coverages_count = db.Column(db.Integer) + senders_count = db.Column(db.Integer) + + # Relations + receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True) + receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("statistics", order_by=date.desc())) + + __table_args__ = (Index('idx_receiver_statistics_uc', 'date', 'receiver_id', 'is_trustworthy', unique=True), ) diff --git a/app/model/receiver_status.py b/app/model/receiver_status.py new file mode 100644 index 0000000..d589dab --- /dev/null +++ b/app/model/receiver_status.py @@ -0,0 +1,48 @@ +from app import db + + +class ReceiverStatus(db.Model): + __tablename__ = "receiver_statuses" + + reference_timestamp = db.Column(db.DateTime, primary_key=True) + + # APRS data + name = db.Column(db.String) + dstcall = db.Column(db.String) + receiver_name = db.Column(db.String(9)) + timestamp = db.Column(db.DateTime) + + # Type information + beacon_type = None + aprs_type = None + + # Debug information + raw_message = None + + # Receiver specific data + version = db.Column(db.String) + platform = db.Column(db.String) + cpu_load = None + free_ram = None + total_ram = None + ntp_error = None + + rt_crystal_correction = None + voltage = None + amperage = None + cpu_temp = db.Column(db.Float(precision=2)) + senders_visible = None + senders_total = None + rec_crystal_correction = None + rec_crystal_correction_fine = None + rec_input_noise = db.Column(db.Float(precision=2)) + senders_signal = None + senders_messages = None + good_senders_signal = None + good_senders = None + good_and_bad_senders = None + + # Calculated values (from this software) + location_mgrs = db.Column(db.String(15)) # full mgrs (15 chars) + location_mgrs_short = db.Column(db.String(9)) # reduced mgrs (9 chars), e.g. used for melissas range tool + agl = db.Column(db.Float(precision=2)) diff --git a/app/model/relation_statistic.py b/app/model/relation_statistic.py new file mode 100644 index 0000000..bb8205b --- /dev/null +++ b/app/model/relation_statistic.py @@ -0,0 +1,26 @@ +from app import db + +from sqlalchemy import Index +from sqlalchemy.orm import backref + + +class RelationStatistic(db.Model): + __tablename__ = "relation_statistics" + + id = db.Column(db.Integer, primary_key=True) + + date = db.Column(db.Date) + is_trustworthy = db.Column(db.Boolean) + + max_distance = db.Column(db.Float(precision=2)) + max_normalized_quality = db.Column(db.Float(precision=2)) + messages_count = db.Column(db.Integer) + + # Relations + sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("relation_stats", order_by=date)) + + receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True) + receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("relation_stats", order_by=date)) + + __table_args__ = (Index('idx_relation_statistics_uc', 'date', 'sender_id', 'receiver_id', 'is_trustworthy', unique=True), ) diff --git a/app/model/device.py b/app/model/sender.py similarity index 70% rename from app/model/device.py rename to app/model/sender.py index 0fa4113..18609e5 100644 --- a/app/model/device.py +++ b/app/model/sender.py @@ -1,19 +1,19 @@ import datetime from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy import Index from app import db -from .device_info import DeviceInfo from app.model.aircraft_type import AircraftType -class Device(db.Model): - __tablename__ = "devices" +class Sender(db.Model): + __tablename__ = "senders" - name = db.Column(db.String, primary_key=True) + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String) - # address = db.Column(db.String(6), index=True) - address = db.Column(db.String, index=True) + address = db.Column(db.String(6), index=True) firstseen = db.Column(db.DateTime, index=True) lastseen = db.Column(db.DateTime, index=True) aircraft_type = db.Column(db.Enum(AircraftType), nullable=False, default=AircraftType.UNKNOWN) @@ -22,21 +22,13 @@ class Device(db.Model): hardware_version = db.Column(db.SmallInteger) real_address = db.Column(db.String(6)) + __table_args__ = (Index('idx_senders_name_uc', 'name', unique=True), ) + def __repr__(self): - return "" % (self.address, self.aircraft_type, self.stealth, self.software_version, self.hardware_version, self.real_address) - - @hybrid_property - def info(self): - query = db.session.query(DeviceInfo).filter(DeviceInfo.address == self.address).order_by(DeviceInfo.address_origin) - - return query.first() - - def get_infos(self): - query = db.session.query(DeviceInfo).filter(DeviceInfo.address == self.address).order_by(DeviceInfo.address_origin) - - return [info for info in query.all()] + return "" % (self.address, self.aircraft_type, self.stealth, self.software_version, self.hardware_version, self.real_address) EXPIRY_DATES = { + 7.01: datetime.date(2022, 2, 28), 7.0: datetime.date(2021, 10, 31), 6.83: datetime.date(2021, 10, 31), 6.82: datetime.date(2021, 5, 31), diff --git a/app/model/sender_direction_statistic.py b/app/model/sender_direction_statistic.py new file mode 100644 index 0000000..5ccc63f --- /dev/null +++ b/app/model/sender_direction_statistic.py @@ -0,0 +1,24 @@ +from app import db + +from sqlalchemy import Index +from sqlalchemy.orm import backref + +from sqlalchemy.dialects.postgresql import JSON + +class SenderDirectionStatistic(db.Model): + __tablename__ = "sender_direction_statistics" + + id = db.Column(db.Integer, primary_key=True) + + directions_count = db.Column(db.Integer) + messages_count = db.Column(db.Integer) + direction_data = db.Column(db.JSON) + + # Relations + sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("direction_stats", order_by=directions_count.desc())) + + receiver_id = db.Column(db.Integer, db.ForeignKey("receivers.id", ondelete="CASCADE"), index=True) + receiver = db.relationship("Receiver", foreign_keys=[receiver_id], backref=backref("direction_stats", order_by=directions_count.desc())) + + __table_args__ = (Index('idx_sender_direction_statistics_uc', 'sender_id', 'receiver_id', unique=True), ) diff --git a/app/model/device_info.py b/app/model/sender_info.py similarity index 53% rename from app/model/device_info.py rename to app/model/sender_info.py index 7b480c7..67aa303 100644 --- a/app/model/device_info.py +++ b/app/model/sender_info.py @@ -1,15 +1,16 @@ from app import db -from .device_info_origin import DeviceInfoOrigin +from .sender_info_origin import SenderInfoOrigin from .aircraft_type import AircraftType +from sqlalchemy.orm import backref +#from sqlalchemy.dialects.postgresql import ENUM -class DeviceInfo(db.Model): - __tablename__ = "device_infos" +class SenderInfo(db.Model): + __tablename__ = "sender_infos" id = db.Column(db.Integer, primary_key=True) + address = db.Column(db.String(6), index=True) address_type = None - # address = db.Column(db.String(6), index=True) - address = db.Column(db.String, index=True) aircraft = db.Column(db.String) registration = db.Column(db.String(7)) competition = db.Column(db.String(3)) @@ -17,10 +18,14 @@ class DeviceInfo(db.Model): identified = db.Column(db.Boolean) aircraft_type = db.Column(db.Enum(AircraftType), nullable=False, default=AircraftType.UNKNOWN) - address_origin = db.Column(db.Enum(DeviceInfoOrigin), nullable=False, default=DeviceInfoOrigin.UNKNOWN) + address_origin = db.Column(db.Enum(SenderInfoOrigin), nullable=False, default=SenderInfoOrigin.UNKNOWN) + + # Relations + sender_id = db.Column(db.Integer, db.ForeignKey("senders.id"), index=True) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("infos", order_by=address_origin)) def __repr__(self): - return "" % ( + return "" % ( self.address_type, self.address, self.aircraft, diff --git a/app/model/sender_info_origin.py b/app/model/sender_info_origin.py new file mode 100644 index 0000000..eb7c4a0 --- /dev/null +++ b/app/model/sender_info_origin.py @@ -0,0 +1,9 @@ +import enum + + +class SenderInfoOrigin(enum.Enum): + # lower number == more trustworthy + USER_DEFINED = 0 + OGN_DDB = 1 + FLARMNET = 2 + UNKNOWN = 3 diff --git a/app/model/sender_position.py b/app/model/sender_position.py new file mode 100644 index 0000000..caf9a24 --- /dev/null +++ b/app/model/sender_position.py @@ -0,0 +1,63 @@ +from geoalchemy2.types import Geometry +from app import db + +from .aircraft_type import AircraftType + + +class SenderPosition(db.Model): + __tablename__ = "sender_positions" + + reference_timestamp = db.Column(db.DateTime, primary_key=True) + + # APRS data + name = db.Column(db.String) + dstcall = db.Column(db.String) + relay = db.Column(db.String) + receiver_name = db.Column(db.String(9)) + timestamp = db.Column(db.DateTime) + location = db.Column("location", Geometry("POINT", srid=4326)) + symboltable = None + symbolcode = None + + track = db.Column(db.SmallInteger) + ground_speed = db.Column(db.Float(precision=2)) + altitude = db.Column(db.Float(precision=2)) + + comment = None + + # Type information + beacon_type = None + aprs_type = None + + # Debug information + raw_message = None + + # Flarm specific data + address_type = db.Column(db.SmallInteger) + aircraft_type = db.Column(db.Enum(AircraftType), nullable=False, default=AircraftType.UNKNOWN) + stealth = db.Column(db.Boolean) + address = db.Column(db.String) + climb_rate = db.Column(db.Float(precision=2)) + turn_rate = db.Column(db.Float(precision=2)) + signal_quality = db.Column(db.Float(precision=2)) + error_count = db.Column(db.SmallInteger) + frequency_offset = db.Column(db.Float(precision=2)) + gps_quality_horizontal = db.Column(db.SmallInteger) + gps_quality_vertical = db.Column(db.SmallInteger) + software_version = db.Column(db.Float(precision=2)) + hardware_version = db.Column(db.SmallInteger) + real_address = db.Column(db.String(6)) + signal_power = db.Column(db.Float(precision=2)) + + #proximity = None + + # Calculated values (from parser) + distance = db.Column(db.Float(precision=2)) + bearing = db.Column(db.SmallInteger) + normalized_quality = db.Column(db.Float(precision=2)) # signal quality normalized to 10km + + # Calculated values (from this software) + location_mgrs = db.Column(db.String(15)) # full mgrs (15 chars) + location_mgrs_short = db.Column(db.String(9)) # reduced mgrs (9 chars), e.g. used for melissas range tool + agl = db.Column(db.Float(precision=2)) + diff --git a/app/model/sender_position_statistic.py b/app/model/sender_position_statistic.py new file mode 100644 index 0000000..6fcf497 --- /dev/null +++ b/app/model/sender_position_statistic.py @@ -0,0 +1,26 @@ +from app import db + +from sqlalchemy import Index +from sqlalchemy.orm import backref + +from .aircraft_type import AircraftType + +from sqlalchemy.dialects.postgresql import ENUM + +class SenderPositionStatistic(db.Model): + __tablename__ = "sender_position_statistics" + + id = db.Column(db.Integer, primary_key=True) + + date = db.Column(db.Date) + + dstcall = db.Column(db.String) + address_type = db.Column(db.SmallInteger) + aircraft_type = db.Column(ENUM(AircraftType, create_type=False), nullable=False, default=AircraftType.UNKNOWN) + stealth = db.Column(db.Boolean) + software_version = db.Column(db.Float(precision=2)) + hardware_version = db.Column(db.SmallInteger) + + messages_count = db.Column(db.Integer) + + __table_args__ = (Index('idx_sender_position_statistics_uc', 'date', 'dstcall', 'address_type', 'aircraft_type', 'stealth', 'software_version', 'hardware_version', unique=True), ) diff --git a/app/model/sender_statistic.py b/app/model/sender_statistic.py new file mode 100644 index 0000000..392d94a --- /dev/null +++ b/app/model/sender_statistic.py @@ -0,0 +1,25 @@ +from app import db + +from sqlalchemy import Index +from sqlalchemy.orm import backref + + +class SenderStatistic(db.Model): + __tablename__ = "sender_statistics" + + id = db.Column(db.Integer, primary_key=True) + + date = db.Column(db.Date) + is_trustworthy = db.Column(db.Boolean) + + max_distance = db.Column(db.Float(precision=2)) + max_normalized_quality = db.Column(db.Float(precision=2)) + messages_count = db.Column(db.Integer) + coverages_count = db.Column(db.Integer) + receivers_count = db.Column(db.Integer) + + # Relations + sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE"), index=True) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref=backref("statistics", order_by=date.desc())) + + __table_args__ = (Index('idx_sender_statistics_uc', 'date', 'sender_id', 'is_trustworthy', unique=True), ) diff --git a/app/model/takeoff_landing.py b/app/model/takeoff_landing.py index 5823080..4f0ff18 100644 --- a/app/model/takeoff_landing.py +++ b/app/model/takeoff_landing.py @@ -1,15 +1,21 @@ from app import db +from sqlalchemy import Index class TakeoffLanding(db.Model): __tablename__ = "takeoff_landings" - address = db.Column(db.String, primary_key=True) - airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="SET NULL"), primary_key=True) - timestamp = db.Column(db.DateTime, primary_key=True) + id = db.Column(db.Integer, primary_key=True) + timestamp = db.Column(db.DateTime) is_takeoff = db.Column(db.Boolean) track = db.Column(db.SmallInteger) # Relations + sender_id = db.Column(db.Integer, db.ForeignKey("senders.id", ondelete="CASCADE")) + sender = db.relationship("Sender", foreign_keys=[sender_id], backref="takeoff_landings") + + airport_id = db.Column(db.Integer, db.ForeignKey("airports.id", ondelete="SET NULL")) airport = db.relationship("Airport", foreign_keys=[airport_id], backref="takeoff_landings") + + __table_args__ = (Index('idx_takeoff_landings_uc', 'timestamp', 'sender_id', 'airport_id', unique=True), ) diff --git a/app/templates/airport_detail.html b/app/templates/airport_detail.html index 0530266..8a75800 100644 --- a/app/templates/airport_detail.html +++ b/app/templates/airport_detail.html @@ -21,21 +21,36 @@
-

Seen Devices

+

Receivers

+ + + + + + {% for receiver in airport.receivers %} + + + + {% endfor %} +
Name
{{ receiver.name }}
+
+ +
+

Seen Senders

- - - - + + + + - {% for device in devices %} + {% for sender in senders %} - - - + + + {% endfor %}
AddressRegistrationLast takeoff/landingSoftware versionNameLast takeoff/landingHardware versionSoftware version
{{ device.address }}{% if device.info is none %}-{% else %}{{ device.info.registration }}{% endif %}{% if device.takeoff_landings %}{% set last_action = device.takeoff_landings|last %}{% if last_action.is_takeoff == True %}↗{% else %}↘{% endif %} @ {{ last_action.timestamp.strftime('%Y-%m-%d %H:%M:%S') }}{% endif %} - {% if device.software_version is not none %}{{ device.software_version }}{% else %}-{% endif %}{{ sender|to_html_link|safe }}{% if sender.takeoff_landings %}{% set last_action = sender.takeoff_landings|last %}{% if last_action.is_takeoff == True %}↗{% else %}↘{% endif %} @ {{ last_action.timestamp.strftime('%Y-%m-%d %H:%M:%S') }}{% endif %} + {% if sender.hardware_version is not none %}{{ sender.hardware_version }}{% else %}-{% endif %}{% if sender.software_version is not none %}{{ sender.software_version }}{% else %}-{% endif %}
diff --git a/app/templates/airports.html b/app/templates/airports.html index 4af1157..912b46f 100644 --- a/app/templates/airports.html +++ b/app/templates/airports.html @@ -23,6 +23,7 @@ + @@ -30,8 +31,9 @@ {% for airport in airports %} - + + + {% endfor %}
#Country Name Logbook (takeoff and landings)
{{ loop.index }} - {{ sel_country }} {{ airport.name }}Logbook{{ sel_country }}{{ airport|to_html_link|safe }}Logbook
diff --git a/app/templates/base.html b/app/templates/base.html index 95de861..ec6a91b 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -5,7 +5,7 @@ {% endblock %} {% block navbar %} - +