funkwhale/.gitlab-ci.yml

variables:
  IMAGE_NAME: funkwhale/funkwhale
  IMAGE: $IMAGE_NAME:$CI_COMMIT_REF_NAME
  IMAGE_LATEST: $IMAGE_NAME:latest
  ALL_IN_ONE_IMAGE_NAME: funkwhale/all-in-one
  ALL_IN_ONE_IMAGE: $ALL_IN_ONE_IMAGE_NAME:$CI_COMMIT_REF_NAME
  ALL_IN_ONE_IMAGE_LATEST: $ALL_IN_ONE_IMAGE_NAME:latest
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
  PYTHONDONTWRITEBYTECODE: "true"
  REVIEW_DOMAIN: preview.funkwhale.audio
  REVIEW_INSTANCE_URL: https://funkwhale.juniorjpdj.pl
  DOCKER_HOST: tcp://docker:2375/
  DOCKER_DRIVER: overlay2
  DOCKER_TLS_CERTDIR: ""
  BUILD_PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7

workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_COMMIT_BRANCH

stages:
  - deploy
  - lint
  - test
  - build
  - publish

review_front:
  interruptible: true
  stage: deploy
  image: node:18-alpine
  when: manual
  allow_failure: true
  variables:
    BASE_URL: /-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/
    VUE_APP_ROUTER_BASE_URL: /-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/
    VUE_APP_INSTANCE_URL: $REVIEW_INSTANCE_URL
    NODE_ENV: review
  before_script:
    - apk add --no-cache jq bash coreutils python3
    - rm -rf front-review
    - mkdir front-review
    - cd front
  script:
    - yarn install
    # this is to ensure we don't have any errors in the output,
    # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169
    - yarn run build --base ./ | tee /dev/stderr | (! grep -i 'ERROR in')
    - cp -r dist/* ../front-review
  artifacts:
    expire_in: 2 weeks
    paths:
      - front-review
  cache:
    key: "funkwhale__front_dependencies"
    paths:
      - front/node_modules
      - front/yarn.lock
  tags:
    - docker
  environment:
    name: review/front/$CI_COMMIT_REF_NAME
    url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/index.html

review_docs:
  interruptible: true
  stage: deploy
  allow_failure: true
  image: python:3.10
  variables:
    BUILD_PATH: "../docs-review"
  before_script:
    - rm -rf docs-review
    - mkdir docs-review
    - cd docs
    - apt-get update
    - apt-get install -y graphviz git
    - pip install poetry
    - poetry install
    - git switch develop && git pull
    - git switch stable && git pull
    - git switch $CI_COMMIT_BRANCH && git pull
  script:
    - poetry run python -m sphinx . $BUILD_PATH
  cache:
    key: "$CI_PROJECT_ID__sphinx"
    paths:
      - "$PIP_CACHE_DIR"
  artifacts:
    expire_in: 2 weeks
    paths:
      - docs-review
  tags:
    - docker
  environment:
    name: review/docs/$CI_COMMIT_REF_NAME
    url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/docs-review/index.html
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - docs/**/*
    - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH
      when: manual

changelog_snippet:
  interruptible: true
  image: alpine:3.17
  stage: lint
  before_script:
    - apk add git
    - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
  script:
    - git diff --name-only FETCH_HEAD | grep "changes/changelog.d/*"
  rules:
    - if: $CI_COMMIT_AUTHOR == 'Renovate Bot <bot@dev.funkwhale.audio>'
      when: never
    - if: $CI_MERGE_REQUEST_TITLE =~ /NOCHANGELOG/
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

pre-commit:
  stage: lint
  image: python:3.10
  variables:
    PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip
    PRE_COMMIT_HOME: $CI_PROJECT_DIR/.cache/pre-commit
  cache:
    paths:
      - $PIP_CACHE_DIR
      - $PRE_COMMIT_HOME
  before_script:
    - pip3 install pre-commit
  script:
    - pre-commit run --all --color=always --show-diff-on-failure

black:
  interruptible: true
  image: python:3.10
  stage: lint
  variables:
    GIT_STRATEGY: fetch
  before_script:
    - pip install black
  script:
    - black --check --diff .
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - api/**/*

flake8:
  interruptible: true
  image: python:3.7
  stage: lint
  variables:
    GIT_STRATEGY: fetch
  before_script:
    - pip install 'flake8<3.7'
  script:
    - flake8 -v api
  cache:
    key: "$CI_PROJECT_ID__flake8_pip_cache"
    paths:
      - "$PIP_CACHE_DIR"
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - api/**/*

eslint:
  interruptible: true
  image: node:18-alpine
  stage: lint
  before_script:
    - cd front
    - apk add --no-cache jq bash coreutils python3
    - yarn install
  script:
    - yarn lint --max-warnings 0
    - yarn lint:tsc
  cache:
    key: "$CI_PROJECT_ID__eslint_npm_cache"
    paths:
      - front/node_modules
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - front/**/*

test_api:
  interruptible: true
  services:
    - postgres:15-alpine
    - redis:7-alpine
  stage: test
  retry: 1
  cache:
    key: "$CI_PROJECT_ID__pip_cache"
    paths:
      - "$PIP_CACHE_DIR"
  variables:
    DATABASE_URL: "postgresql://postgres@postgres/postgres"
    FUNKWHALE_URL: "https://funkwhale.ci"
    DJANGO_SETTINGS_MODULE: config.settings.local
    POSTGRES_HOST_AUTH_METHOD: trust
    CACHE_URL: "redis://redis:6379/0"
  before_script:
    - cd api
    - poetry install --no-root
  script:
    - poetry run pytest --cov-report xml --cov-report term-missing:skip-covered --cov=funkwhale_api --junitxml=report.xml tests/
  tags:
    - docker
  artifacts:
    expire_in: 2 weeks
    reports:
      junit: api/report.xml
      coverage_report:
        coverage_format: cobertura
        path: api/coverage.xml
  parallel:
    matrix:
      - PY_VER: ["3.7", "3.8", "3.9", "3.10", "3.11"]
  image: $CI_REGISTRY/funkwhale/backend-test-docker:$PY_VER
  coverage: '/TOTAL\s*\d*\s*\d*\s*(\d*%)/'
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push"
      changes:
        - api/**/*
    - if: $CI_COMMIT_REF_PROTECTED == "true"
      when: always

# Those tests are disabled for now since no vitest dom emulation is providing
# AudioContext, which is required for our HTML audio player
#test_front:
#  interruptible: true
#  stage: test
#  image: node:18-alpine
#  before_script:
#    - cd front
#    - apk add --no-cache jq bash coreutils python3
#  script:
#    - yarn install --check-files
#    - yarn test:unit
#  cache:
#    key: "funkwhale__front_dependencies"
#    paths:
#      - front/node_modules
#      - front/yarn.lock
#  artifacts:
#    name: "front_${CI_COMMIT_REF_NAME}"
#    paths:
#      - front/dist/
#    reports:
#      junit: front/coverage/cobertura-coverage.xml
#  tags:
#    - docker
#  rules:
#    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push"
#      changes:
#        - front/**/*
#    - if: $CI_COMMIT_REF_PROTECTED == "true"
#      when: always

build_docs:
  stage: build
  image: $CI_REGISTRY/funkwhale/backend-test-docker:3.10
  services:
    - postgres:15-alpine
    - redis:7-alpine
  cache:
    key: "$CI_PROJECT_ID__pip_cache"
    paths:
      - "$PIP_CACHE_DIR"
  variables:
    DATABASE_URL: "postgresql://postgres@postgres/postgres"
    FUNKWHALE_URL: "https://funkwhale.ci"
    DJANGO_SETTINGS_MODULE: config.settings.local
    POSTGRES_HOST_AUTH_METHOD: trust
    CACHE_URL: "redis://redis:6379/0"
    API_TYPE: "v1"
  before_script:
    - cd api
    - pip3 install poetry
    - poetry install
    - poetry run python manage.py migrate
  script:
    - poetry run python manage.py spectacular --file ../docs/schema.yml
  tags:
    - docker
  artifacts:
    expire_in: 2 weeks
    paths:
      - docs/schema.yml

build_front:
  stage: build
  image: node:18-alpine
  before_script:
    - apk add --no-cache jq bash coreutils python3
    - cd front
  script:
    - yarn install
    - yarn run i18n-compile
    # this is to ensure we don't have any errors in the output,
    # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169
    - yarn run build:deployment | tee /dev/stderr | (! grep -i 'ERROR in')
    - chmod -R 755 dist
  artifacts:
    name: "front_${CI_COMMIT_REF_NAME}"
    paths:
      - front/dist/
  only:
    - tags@funkwhale/funkwhale
    - stable@funkwhale/funkwhale
    - develop@funkwhale/funkwhale
  tags:
    - docker

build_documentation:
  stage: build
  image: python:3.10
  variables:
    BUILD_PATH: "../public"
  before_script:
    - cd docs
    - apt-get update
    - apt-get install -y graphviz git
    - pip install poetry
    - poetry install
    - git switch develop && git pull
    - git switch stable && git pull
    - git switch $CI_COMMIT_BRANCH && git pull
  script:
    - ./build_docs.sh
  cache:
    key: "$CI_PROJECT_ID__sphinx"
    paths:
      - "$PIP_CACHE_DIR"
  artifacts:
    expire_in: 2 weeks
    paths:
      - public
  rules:
    - if: $CI_COMMIT_BRANCH == "stable"
      when: always
    - if: $CI_COMMIT_BRANCH == "develop"
      changes:
        - docs/**/*
      when: always
  tags:
    - docker

deploy_documentation:
  stage: publish
  image: alpine
  dependencies:
    - build_documentation
  before_script:
    - apk add openssh-client rsync
    - mkdir -p ~/.ssh
    - echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - eval `ssh-agent -s`
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
  script:
    - rsync -r -e "ssh -p 2282" $CI_PROJECT_DIR/public/ docs@docs.funkwhale.audio:/htdocs/
  only:
    - stable

.docker_publish:
  stage: publish
  image: egon0/docker-with-buildx-and-git:bash
  tags:
    - multiarch
  services:
    - docker:20-dind
  before_script:
    - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
  cache:
    key: docker_public_${CI_COMMIT_REF_NAME}
    paths:
      - ~/.cargo
  script:

docker_publish_stable_release:
  # Publish a docker image for releases
  extends: .docker_publish
  variables:
    IMAGE_NAME: funkwhale/$COMPONENT
  rules:
    - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^[0-9]+(.[0-9]+){1,2}$/
  script:
    # Check if this is the latest release
    - ./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG - && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=;
    - export major="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1)"
    - export minor="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1,2)"
    - cd $COMPONENT
    - docker buildx create --use --name A$CI_COMMIT_SHORT_SHA
    - docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE $DOCKER_LATEST_TAG -t $IMAGE_NAME:$major -t $IMAGE_NAME:$minor .
  parallel:
    matrix:
      - COMPONENT: ["api", "front"]

docker_publish_unstable_release:
  # Publish a docker image for releases
  extends: .docker_publish
  variables:
    IMAGE_NAME: funkwhale/$COMPONENT
  rules:
    - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME !~ /^[0-9]+(.[0-9]+){1,2}$/
  script:
    # Check if this is the latest release
    - cd $COMPONENT
    - docker buildx create --use --name A$CI_COMMIT_SHORT_SHA
    - docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE .
  parallel:
    matrix:
      - COMPONENT: ["api", "front"]

docker_publish_non-release:
  # Publish a docker image for each commit on develop
  extends: .docker_publish
  variables:
    IMAGE_NAME: funkwhale/$COMPONENT
  only:
    - develop@funkwhale/funkwhale
    - stable@funkwhale/funkwhale
  script:
    - ./scripts/set-api-build-metadata.sh $CI_COMMIT_SHORT_SHA
    - cd $COMPONENT
    - docker buildx create --use --name A$CI_COMMIT_SHORT_SHA
    - docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE .
  parallel:
    matrix:
      - COMPONENT: ["api", "front"]

docker_all_in_one_release:
  stage: publish
  image: egon0/docker-with-buildx-and-git:bash
  allow_failure: true
  services:
    - docker:20-dind
  variables:
    ALL_IN_ONE_REF: main
    ALL_IN_ONE_ARTIFACT_URL: https://dev.funkwhale.audio/funkwhale/funkwhale-docker-all-in-one/-/archive/$ALL_IN_ONE_REF/funkwhale-docker-all-in-one-$ALL_IN_ONE_REF.zip
    BUILD_PATH: all_in_one
  before_script:
    - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
  script:
    - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
    - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
    - unzip -o all_in_one.zip -d tmpdir
    - mv tmpdir/funkwhale-docker-all-in-one-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
    - cp -r api $BUILD_PATH/src/api
    - cp -r front $BUILD_PATH/src/front
    - cd $BUILD_PATH
    - ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
    - docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG .
    - docker push $ALL_IN_ONE_IMAGE
    - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi
  only:
    - develop@funkwhale/funkwhale
    - stable@funkwhale/funkwhale
    - tags@funkwhale/funkwhale

build_api:
  # Simply publish a zip containing api/ directory
  stage: publish
  image: bash
  artifacts:
    name: "api_${CI_COMMIT_REF_NAME}"
    paths:
      - api
  script:
    - rm -rf api/tests
    - >
      if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ]; then
        ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8);
      fi
    - chmod -R 750 api
  only:
    - tags@funkwhale/funkwhale
    - stable@funkwhale/funkwhale
    - develop@funkwhale/funkwhale