From 6c0a43a0eaaf40bb9ab16db2775f706d02098ebb Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 18:19:32 +0200 Subject: [PATCH] We can now work on federation locally thank to traefik --- .env.dev | 6 ++-- README.rst | 67 +++++++++++++++++++++++++++++++++++ api/config/settings/common.py | 27 ++++++++++++-- dev.yml | 45 ++++++++++++++++++++--- docker/nginx/entrypoint.sh | 15 +++++--- docker/ssl/test.crt | 22 ++++++++++++ docker/ssl/test.key | 28 +++++++++++++++ docker/traefik.toml | 26 ++++++++++++++ docker/traefik.yml | 22 ++++++++++++ 9 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 docker/ssl/test.crt create mode 100644 docker/ssl/test.key create mode 100644 docker/traefik.toml create mode 100644 docker/traefik.yml diff --git a/.env.dev b/.env.dev index 2e8834143..5a010cdf4 100644 --- a/.env.dev +++ b/.env.dev @@ -1,9 +1,11 @@ API_AUTHENTICATION_REQUIRED=True RAVEN_ENABLED=false RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5 -DJANGO_ALLOWED_HOSTS=localhost,nginx +DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx DJANGO_SETTINGS_MODULE=config.settings.local DJANGO_SECRET_KEY=dev C_FORCE_ROOT=true -FUNKWHALE_URL=http://localhost +FUNKWHALE_HOSTNAME=localhost +FUNKWHALE_PROTOCOL=http PYTHONDONTWRITEBYTECODE=true +WEBPACK_DEVSERVER_PORT=8080 diff --git a/README.rst b/README.rst index 2d5d2011d..2e4772adb 100644 --- a/README.rst +++ b/README.rst @@ -206,3 +206,70 @@ Typical workflow for a merge request 6. Push your branch 7. Create your merge request 8. Take a step back and enjoy, we're really grateful you did all of this and took the time to contribute! + + +Working with federation locally +------------------------------- + +To achieve that, you'll need: + +1. to update your dns resolver to resolve all your .dev hostnames locally +2. a reverse proxy (such as traefik) to catch those .dev requests and + and with https certificate +3. two instances (or more) running locally, following the regular dev setup + +Resolve .dev names locally +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you use dnsmasq, this is as simple as doing:: + + echo "address=/test/172.17.0.1" | sudo tee /etc/dnsmasq.d/test.conf + sudo systemctl restart dnsmasq + +If you use NetworkManager with dnsmasq integration, use this instead:: + + echo "address=/test/172.17.0.1" | sudo tee /etc/NetworkManager/dnsmasq.d/test.conf + sudo systemctl restart NetworkManager + +Add wildcard certificate to the trusted certificates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Simply copy bundled certificates:: + + sudo cp docker/ssl/test.crt /usr/local/share/ca-certificates/ + sudo update-ca-certificates + +This certificate is a wildcard for ``*.funkwhale.test`` + +Run a reverse proxy for your instances +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + +Crete docker network +^^^^^^^^^^^^^^^^^^^^ + +Create the federation network:: + + docker network create federation + +Launch everything +^^^^^^^^^^^^^^^^^ + +Launch the traefik proxy:: + + docker-compose -f docker/traefik.yml up -d + +Then, in separate terminals, you can setup as many different instances as you +need:: + + export COMPOSE_PROJECT_NAME=node2 + docker-compose -f dev.yml run --rm api python manage.py migrate + docker-compose -f dev.yml run --rm api python manage.py createsuperuser + docker-compose -f dev.yml up nginx api front + +Note that by default, if you don't export the COMPOSE_PROJECT_NAME, +we will default to node1 as the name of your instance. + +Assuming your project name is ``node1``, your server will be reachable +at ``https://node1.funkwhale.test/``. Not that you'll have to trust +the SSL Certificate as it's self signed. diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 7b7f6e64c..ebfd21dd6 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -25,8 +25,26 @@ try: except FileNotFoundError: pass -FUNKWHALE_URL = env('FUNKWHALE_URL') -FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc +FUNKWHALE_HOSTNAME = None +FUNKWHALE_HOSTNAME_SUFFIX = env('FUNKWHALE_HOSTNAME_SUFFIX', default=None) +FUNKWHALE_HOSTNAME_PREFIX = env('FUNKWHALE_HOSTNAME_PREFIX', default=None) +if FUNKWHALE_HOSTNAME_PREFIX and FUNKWHALE_HOSTNAME_SUFFIX: + # We're in traefik case, in development + FUNKWHALE_HOSTNAME = '{}.{}'.format( + FUNKWHALE_HOSTNAME_PREFIX, FUNKWHALE_HOSTNAME_SUFFIX) + FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https') +else: + try: + FUNKWHALE_HOSTNAME = env('FUNKWHALE_HOSTNAME') + FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https') + except Exception: + FUNKWHALE_URL = env('FUNKWHALE_URL') + _parsed = urlsplit(FUNKWHALE_URL) + FUNKWHALE_HOSTNAME = _parsed.netloc + FUNKWHALE_PROTOCOL = _parsed.scheme + +FUNKWHALE_URL = '{}://{}'.format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME) + FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True) FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME) @@ -406,3 +424,8 @@ ACCOUNT_USERNAME_BLACKLIST = [ 'staff', 'service', ] + env.list('ACCOUNT_USERNAME_BLACKLIST', default=[]) + +EXTERNAL_REQUESTS_VERIFY_SSL = env.bool( + 'EXTERNAL_REQUESTS_VERIFY_SSL', + default=True +) diff --git a/dev.yml b/dev.yml index 409b6b4a7..9488d4a6f 100644 --- a/dev.yml +++ b/dev.yml @@ -10,22 +10,39 @@ services: - "HOST=0.0.0.0" - "WEBPACK_DEVSERVER_PORT=${WEBPACK_DEVSERVER_PORT-8080}" ports: - - "${WEBPACK_DEVSERVER_PORT-8080}:${WEBPACK_DEVSERVER_PORT-8080}" + - "${WEBPACK_DEVSERVER_PORT_BINDING-8080:}${WEBPACK_DEVSERVER_PORT-8080}" volumes: - './front:/app' - './po:/po' + networks: + - federation + - internal + labels: + traefik.backend: "${COMPOSE_PROJECT_NAME-node1}" + traefik.frontend.rule: "Host: ${COMPOSE_PROJECT_NAME-node1}.funkwhale.test" + traefik.enable: 'true' + traefik.federation.protocol: 'http' + traefik.federation.port: "${WEBPACK_DEVSERVER_PORT-8080}" postgres: env_file: - .env.dev - .env image: postgres + volumes: + - "./data/${COMPOSE_PROJECT_NAME-node1}/postgres:/var/lib/postgresql/data" + networks: + - internal redis: env_file: - .env.dev - .env image: redis:3.0 + volumes: + - "./data/${COMPOSE_PROJECT_NAME-node1}/redis:/data" + networks: + - internal celeryworker: env_file: @@ -39,11 +56,17 @@ services: - redis command: celery -A funkwhale_api.taskapp worker -l debug environment: + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" + - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test" + - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}" + - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL-http}" - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" volumes: - ./api:/app - ./data/music:/music + networks: + - internal api: env_file: - .env.dev @@ -56,12 +79,17 @@ services: - ./api:/app - ./data/music:/music environment: + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" + - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test" + - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}" + - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL-http}" - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" links: - postgres - redis - + networks: + - internal nginx: command: /entrypoint.sh env_file: @@ -70,6 +98,8 @@ services: image: nginx environment: - "WEBPACK_DEVSERVER_PORT=${WEBPACK_DEVSERVER_PORT-8080}" + - "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME- }" + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" links: - api - front @@ -79,8 +109,9 @@ services: - ./deploy/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf.template:ro - ./api/funkwhale_api/media:/protected/media ports: - - "0.0.0.0:6001:6001" - + - "6001" + networks: + - internal docs: build: docs command: python serve.py @@ -89,3 +120,9 @@ services: ports: - '35730:35730' - '8001:8001' + +networks: + internal: + federation: + external: + name: federation diff --git a/docker/nginx/entrypoint.sh b/docker/nginx/entrypoint.sh index 1819acf1c..14e072a7e 100755 --- a/docker/nginx/entrypoint.sh +++ b/docker/nginx/entrypoint.sh @@ -1,10 +1,17 @@ #!/bin/bash -eux -FIRST_HOST=$(echo ${DJANGO_ALLOWED_HOSTS} | cut -d, -f1) + +FORWARDED_PORT="$WEBPACK_DEVSERVER_PORT" +COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME// /}" +if [ -n "$COMPOSE_PROJECT_NAME" ]; then + echo + FUNKWHALE_HOSTNAME="$COMPOSE_PROJECT_NAME.funkwhale.test" + FORWARDED_PORT="443" +fi echo "Copying template file..." cp /etc/nginx/funkwhale_proxy.conf{.template,} -sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FIRST_HOST}:${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf -sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FIRST_HOST}/" /etc/nginx/funkwhale_proxy.conf -sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FUNKWHALE_HOSTNAME}:${FORWARDED_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FUNKWHALE_HOSTNAME}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${FORWARDED_PORT}/" /etc/nginx/funkwhale_proxy.conf cat /etc/nginx/funkwhale_proxy.conf nginx -g "daemon off;" diff --git a/docker/ssl/test.crt b/docker/ssl/test.crt new file mode 100644 index 000000000..e4d3eefb8 --- /dev/null +++ b/docker/ssl/test.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDljCCAn6gAwIBAgIJAOA/w9NwL3aMMA0GCSqGSIb3DQEBCwUAMGAxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQxGTAXBgNVBAMMECouZnVua3doYWxlLnRlc3QwHhcNMTgw +NDA4MTMwNDAzWhcNMjgwNDA1MTMwNDAzWjBgMQswCQYDVQQGEwJBVTETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRkwFwYDVQQDDBAqLmZ1bmt3aGFsZS50ZXN0MIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAyGqRLEMFs1mpRwauTicIRj2zwBUe6JMNRbIvOUkaj2KY6avA +7tiNti/ygBoTyJl2JK3mmLqxElqedpMhjVvYde/PyjXoZ+0Vq4FWv89LV6ZM/Scf +TCIYwWF1ppi6GYFmU3WCIMISkKiPBtMArB0oZxiUWLmkyd8jih2wnQOpkQ20FfG0 +CtlrKlQKyAe7X3zPuqGfaMUN7J4w9g3/SC66YulbAtI1/Z4tuG8J4m2RC6jH1hVy +364l3ifEC+m9Kax/ystfu/mkLdyQgRfOZTNf2JhS3BL8zpoWMXFK+4+7TYisrV1h +0pzIAsoQeBB+cFOOFEwRAv0FxSWnZ+/shjnwbwIDAQABo1MwUTAdBgNVHQ4EFgQU +sULmofttRyWUMM93IsD8jBvyCd4wHwYDVR0jBBgwFoAUsULmofttRyWUMM93IsD8 +jBvyCd4wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAUg/fiXut +hW6fDx9f0JdB4uLiLnv8tDP35ackLLapFJhXtflIXcqCzxStQ46nMs1wjaZPb+ws +pLULzvTKTxJbu+JYc2nvis4m2oSFczJ3S9tgug4Ppv8yS7N1pp7kfjOvBjgh6sYW +p+Ctb5r8qvgvT9yDTeCnsqktb/OkRHlHwhRYfnuxh+96s4mzifqFUP4uCCcFYPTc +RE0Ag3oI5sHOdDk/cdYE5PGQPjSP6gzn0lsrz1Q3x1C8+txSHzsJnvS3Ost+dwcy +JSjDBXauy9cZv93Voevcl16Ioo7trtkp4dwAoep52vOT/KMkJ4zm19msV3BP4wMa +BUqrV2F7twD5zw== +-----END CERTIFICATE----- diff --git a/docker/ssl/test.key b/docker/ssl/test.key new file mode 100644 index 000000000..669e5f700 --- /dev/null +++ b/docker/ssl/test.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDIapEsQwWzWalH +Bq5OJwhGPbPAFR7okw1Fsi85SRqPYpjpq8Du2I22L/KAGhPImXYkreaYurESWp52 +kyGNW9h178/KNehn7RWrgVa/z0tXpkz9Jx9MIhjBYXWmmLoZgWZTdYIgwhKQqI8G +0wCsHShnGJRYuaTJ3yOKHbCdA6mRDbQV8bQK2WsqVArIB7tffM+6oZ9oxQ3snjD2 +Df9ILrpi6VsC0jX9ni24bwnibZELqMfWFXLfriXeJ8QL6b0prH/Ky1+7+aQt3JCB +F85lM1/YmFLcEvzOmhYxcUr7j7tNiKytXWHSnMgCyhB4EH5wU44UTBEC/QXFJadn +7+yGOfBvAgMBAAECggEBAMVB3lEqRloYTbxSnwzc7g/0ew77usg+tDl8/23qvfGS +od6b5fEvw4sl9hCPmhk+skG3x9dbKR1fg8hBWCzB0XOC7YmhNXXUrBd53eA8L3O9 +gtlHwE424Ra0zg+DEug3rHdImSOU4KDwxpV46Jh+ul1+m8QYNFFdBqXSQxrHmAXj +MQ6++rjoJ+bhucmjBouzMYXHTGhdae3kjDFrFJ4cUsH6F03NcDwS+AmZxa/DWQ/H +SoBQBeLoE6I1aKhLgY91yO1e7CtSzS2GFCODReN4b3cylaR7jE7Mg87TZcga6Wfa +Xcd120VVlVq6HmZc/Xob7aUim3AuY2er8bcvmg1XOsECgYEA5EMM5UlpLdNWv1hp +5IMvkeCbXtLJ3IOHO0xLkFdx0CxaR9TyAAqIrSh1t9rFhYqLUNiOdMc2TqrvdgEU +B/QZrAevWRc5sjPvFXmYeWSCi/tjRgQh4jClWDX/TlfAlP55z2BFyMPMX6//WbBQ +5aL9xymTymzFFcaE8EytT5Jz8rUCgYEA4MVF3IkaQepl6H1gf2T6ev+MtGk9AGg9 +DSJpio7hfMcY5X3NrTJJFF9DJFXqfo3ILOMyUpIUHqkCGKXil0n9ypLp4vq7l+6c +m1gtKFXh7uKAV4XtSnR0nuK/N10JJp2HbbFYGlziRaa1iEPAFvLDQHu4jyf5sXyV +HvreuQgGWRMCgYEAlUaQKWaP5UsfoPUGE04DjwfvM9zv7EkL6CimBhhZswU+aVmG +haZd6bfa/EiTAhkvsMheqVoaVuoMvgRIgEcPfuRrtPyuW68A/O9PWpvzj+3v5zsO +maisiPqPI0HaDNY6/PZ9zKTXhABKIvJehT7JbjTvlOL7JJl2GNxcPvyM3T0CgYEA +tnVtUKi69+ce8qtUOhXufwoTXiBPtJTpelAE/MUfpfq46xJEc+PuDuuFxWk5AaJ2 +bHnBz+VlD76CRR/j4IvfySGZWvfOcHbyCeh6P9P3o8OaC3JcPaRrRs8qCfcsBny6 +AwGDU2MzCvdZRVQ6CmbmuOG13//DYaCQLKXZRrqM7KECgYEAxDsqtyHA/a38UhS8 +iQ8HqrZp8CuzJoJw/QILvzjojD1cvmwF73RrPEpRfEaLWVQGQ5F1IlHk/009C5zy +eUT4ZaPxLem6khBf7pn3xXaVBGZsYoltek5sUBsu/jA+4Sw6bcUmhBRBCs98JGpR +DVJtvOTk9aGW8M8UbgqwW+e/6ng= +-----END PRIVATE KEY----- diff --git a/docker/traefik.toml b/docker/traefik.toml new file mode 100644 index 000000000..85da2ea72 --- /dev/null +++ b/docker/traefik.toml @@ -0,0 +1,26 @@ +defaultEntryPoints = ["http", "https"] + +################################################################ +# Web configuration backend +################################################################ +[web] +address = ":8040" +################################################################ +# Docker configuration backend +################################################################ +[docker] +domain = "funkwhale.test" +watch = true +exposedbydefault = false + +[entryPoints] + [entryPoints.http] + address = ":80" + [entryPoints.http.redirect] + entryPoint = "https" + [entryPoints.https] + address = ":443" + [entryPoints.https.tls] + [[entryPoints.https.tls.certificates]] + certFile = "/ssl/traefik.crt" + keyFile = "/ssl/traefik.key" diff --git a/docker/traefik.yml b/docker/traefik.yml new file mode 100644 index 000000000..0b15b3290 --- /dev/null +++ b/docker/traefik.yml @@ -0,0 +1,22 @@ +version: '2.1' + +services: + traefik: + image: traefik:alpine + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./traefik.toml:/traefik.toml + - ./ssl/test.key:/ssl/traefik.key + - ./ssl/test.crt:/ssl/traefik.crt + ports: + - '80:80' + - '443:443' + - '8040:8040' + networks: + federation: + + +networks: + federation: + external: + name: federation