From 36a0a4df4f6e355e7451963312d09c4ed6b909fe Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 3 Jun 2018 18:33:28 +0200 Subject: [PATCH] Removed transcoding support (#271) --- api/funkwhale_api/music/forms.py | 23 ----------- api/funkwhale_api/music/views.py | 37 ------------------ changes/changelog.d/271.enhancement | 57 ++++++++++++++++++++++++++++ deploy/nginx.conf | 38 ------------------- docker/nginx/conf.dev | 33 ---------------- docs/installation/index.rst | 10 ++--- front/src/components/audio/Track.vue | 8 ---- 7 files changed, 60 insertions(+), 146 deletions(-) delete mode 100644 api/funkwhale_api/music/forms.py create mode 100644 changes/changelog.d/271.enhancement diff --git a/api/funkwhale_api/music/forms.py b/api/funkwhale_api/music/forms.py deleted file mode 100644 index e68ab73cc..000000000 --- a/api/funkwhale_api/music/forms.py +++ /dev/null @@ -1,23 +0,0 @@ -from django import forms - -from . import models - - -class TranscodeForm(forms.Form): - FORMAT_CHOICES = [ - ('ogg', 'ogg'), - ('mp3', 'mp3'), - ] - - to = forms.ChoiceField(choices=FORMAT_CHOICES) - BITRATE_CHOICES = [ - (64, '64'), - (128, '128'), - (256, '256'), - ] - bitrate = forms.ChoiceField( - choices=BITRATE_CHOICES, required=False) - - track_file = forms.ModelChoiceField( - queryset=models.TrackFile.objects.exclude(audio_file__isnull=True) - ) diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index 2f5b75a97..2850c0770 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -35,7 +35,6 @@ from funkwhale_api.musicbrainz import api from funkwhale_api.requests.models import ImportRequest from . import filters -from . import forms from . import importers from . import models from . import permissions as music_permissions @@ -324,42 +323,6 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet): except models.TrackFile.DoesNotExist: return Response(status=404) - @list_route(methods=['get']) - def viewable(self, request, *args, **kwargs): - return Response({}, status=200) - - @list_route(methods=['get']) - def transcode(self, request, *args, **kwargs): - form = forms.TranscodeForm(request.GET) - if not form.is_valid(): - return Response(form.errors, status=400) - - f = form.cleaned_data['track_file'] - if not f.audio_file: - return Response(status=400) - output_kwargs = { - 'format': form.cleaned_data['to'] - } - args = (ffmpeg - .input(f.audio_file.path) - .output('pipe:', **output_kwargs) - .get_args() - ) - # we use a generator here so the view return immediatly and send - # file chunk to the browser, instead of blocking a few seconds - def _transcode(): - p = subprocess.Popen( - ['ffmpeg'] + args, - stdout=subprocess.PIPE) - for line in p.stdout: - yield line - - response = StreamingHttpResponse( - _transcode(), status=200, - content_type=form.cleaned_data['to']) - - return response - class TagViewSet(viewsets.ReadOnlyModelViewSet): queryset = Tag.objects.all().order_by('name') diff --git a/changes/changelog.d/271.enhancement b/changes/changelog.d/271.enhancement new file mode 100644 index 000000000..5db9ec341 --- /dev/null +++ b/changes/changelog.d/271.enhancement @@ -0,0 +1,57 @@ +Removed transcoding support (#271) + + +Removed alpha-state transcoding (#271) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A few months ago, a basic transcoding feature was implemented. Due to the way +this feature was designed, it was slow, CPU intensive on the server side, +and very tightly coupled to the reverse-proxy configuration, preventing +it to work Apache2, for instance. It was also not compatible with Subsonic clients. + +Based on that, we're currently removing support for transcoding +**in its current state**. The work on a better designed transcoding feature +can be tracked in https://code.eliotberriot.com/funkwhale/funkwhale/issues/272. + +You don't have to do anything on your side, but you may want to remove +the now obsolete configuration from your reverse proxy file (nginx only):: + + # Remove those blocks: + + # transcode cache + proxy_cache_path /tmp/funkwhale-transcode levels=1:2 keys_zone=transcode:10m max_size=1g inactive=7d; + + # Transcoding logic and caching + location = /transcode-auth { + include /etc/nginx/funkwhale_proxy.conf; + # needed so we can authenticate transcode requests, but still + # cache the result + internal; + set $query ''; + # ensure we actually pass the jwt to the underlytin auth url + if ($request_uri ~* "[^\?]+\?(.*)$") { + set $query $1; + } + proxy_pass http://funkwhale-api/api/v1/trackfiles/viewable/?$query; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + } + + location /api/v1/trackfiles/transcode/ { + include /etc/nginx/funkwhale_proxy.conf; + # this block deals with authenticating and caching transcoding + # requests. Caching is heavily recommended as transcoding + # is a CPU intensive process. + auth_request /transcode-auth; + if ($args ~ (.*)jwt=[^&]*(.*)) { + set $cleaned_args $1$2; + } + proxy_cache_key "$scheme$request_method$host$uri$is_args$cleaned_args"; + proxy_cache transcode; + proxy_cache_valid 200 7d; + proxy_ignore_headers "Set-Cookie"; + proxy_hide_header "Set-Cookie"; + add_header X-Cache-Status $upstream_cache_status; + proxy_pass http://funkwhale-api; + } + # end of transcoding logic diff --git a/deploy/nginx.conf b/deploy/nginx.conf index 5314d9017..66851321f 100644 --- a/deploy/nginx.conf +++ b/deploy/nginx.conf @@ -1,8 +1,5 @@ # Ensure you update at least the server_name variables to match your own -# transcode cache -proxy_cache_path /tmp/funkwhale-transcode levels=1:2 keys_zone=transcode:10m max_size=1g inactive=7d; - # domain upstream funkwhale-api { # depending on your setup, you may want to udpate this @@ -98,41 +95,6 @@ server { alias /srv/funkwhale/data/music; } - # Transcoding logic and caching - location = /transcode-auth { - include /etc/nginx/funkwhale_proxy.conf; - # needed so we can authenticate transcode requests, but still - # cache the result - internal; - set $query ''; - # ensure we actually pass the jwt to the underlytin auth url - if ($request_uri ~* "[^\?]+\?(.*)$") { - set $query $1; - } - proxy_pass http://funkwhale-api/api/v1/trackfiles/viewable/?$query; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - } - - location /api/v1/trackfiles/transcode/ { - include /etc/nginx/funkwhale_proxy.conf; - # this block deals with authenticating and caching transcoding - # requests. Caching is heavily recommended as transcoding - # is a CPU intensive process. - auth_request /transcode-auth; - if ($args ~ (.*)jwt=[^&]*(.*)) { - set $cleaned_args $1$2; - } - proxy_cache_key "$scheme$request_method$host$uri$is_args$cleaned_args"; - proxy_cache transcode; - proxy_cache_valid 200 7d; - proxy_ignore_headers "Set-Cookie"; - proxy_hide_header "Set-Cookie"; - add_header X-Cache-Status $upstream_cache_status; - proxy_pass http://funkwhale-api; - } - # end of transcoding logic - location /staticfiles/ { # django static files alias /srv/funkwhale/data/static/; diff --git a/docker/nginx/conf.dev b/docker/nginx/conf.dev index 673edd1a4..2ed1a97d5 100644 --- a/docker/nginx/conf.dev +++ b/docker/nginx/conf.dev @@ -26,7 +26,6 @@ http { keepalive_timeout 65; #gzip on; - proxy_cache_path /tmp/funkwhale-transcode levels=1:2 keys_zone=transcode:10m max_size=1g inactive=24h use_temp_path=off; map $http_upgrade $connection_upgrade { default upgrade; @@ -46,38 +45,6 @@ http { internal; alias /music; } - location = /transcode-auth { - # needed so we can authenticate transcode requests, but still - # cache the result - internal; - set $query ''; - # ensure we actually pass the jwt to the underlytin auth url - if ($request_uri ~* "[^\?]+\?(.*)$") { - set $query $1; - } - include /etc/nginx/funkwhale_proxy.conf; - proxy_pass http://api:12081/api/v1/trackfiles/viewable/?$query; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - } - - location /api/v1/trackfiles/transcode/ { - # this block deals with authenticating and caching transcoding - # requests. Caching is heavily recommended as transcoding - # is a CPU intensive process. - auth_request /transcode-auth; - if ($args ~ (.*)jwt=[^&]*(.*)) { - set $cleaned_args $1$2; - } - include /etc/nginx/funkwhale_proxy.conf; - proxy_cache_key "$scheme$request_method$host$uri$is_args$cleaned_args"; - proxy_cache transcode; - proxy_cache_valid 200 7d; - proxy_ignore_headers "Set-Cookie"; - proxy_hide_header "Set-Cookie"; - add_header X-Cache-Status $upstream_cache_status; - proxy_pass http://api:12081; - } location / { include /etc/nginx/funkwhale_proxy.conf; proxy_pass http://api:12081/; diff --git a/docs/installation/index.rst b/docs/installation/index.rst index ae5794b6c..123e3f5ea 100644 --- a/docs/installation/index.rst +++ b/docs/installation/index.rst @@ -16,10 +16,8 @@ The project relies on the following components and services to work: Hardware requirements --------------------- -Funkwhale is not especially CPU hungry, unless you're relying heavily -on the transcoding feature (which is basic and unoptimized at the moment). - -On a dockerized instance with 2 CPUs and a few active users, the memory footprint is around ~500Mb:: +Funkwhale is not especially CPU hungry. On a dockerized instance with 2 CPUs +and a few active users, the memory footprint is around ~500Mb:: CONTAINER MEM USAGE funkwhale_api_1 202.1 MiB @@ -125,10 +123,8 @@ Apache2 are not working yet: - Websocket (used for real-time updates on Instance timeline) - - Transcoding of audio files - Those features are not necessary to use your Funkwhale instance, and - transcoding in particular is still in alpha-state anyway. + Those features are not necessary to use your Funkwhale instance. Ensure you have a recent version of apache2 installed on your server. You'll also need the following dependencies:: diff --git a/front/src/components/audio/Track.vue b/front/src/components/audio/Track.vue index 08a055f5c..366f104f1 100644 --- a/front/src/components/audio/Track.vue +++ b/front/src/components/audio/Track.vue @@ -18,7 +18,6 @@