From b2f4dea8f3d8045bd84fef95e0a5a200ccb95660 Mon Sep 17 00:00:00 2001
From: kompotkot <zenitsws@gmail.com>
Date: Tue, 30 Nov 2021 14:29:10 +0000
Subject: [PATCH] Docker and docker compose setup for db and moonstream api

---
 README.md                              | 22 ++++++++++
 backend/Dockerfile                     | 20 +++++++++
 backend/configs/docker_generate_env.sh | 19 +++++++++
 backend/dev.sh                         | 18 ++++++--
 db/Dockerfile                          | 18 ++++++++
 db/configs/docker_generate_env.sh      | 26 ++++++++++++
 db/migrate.sh                          | 16 ++++++++
 docker-compose.yml                     | 57 ++++++++++++++++++++++++++
 8 files changed, 192 insertions(+), 4 deletions(-)
 create mode 100644 backend/Dockerfile
 create mode 100755 backend/configs/docker_generate_env.sh
 create mode 100644 db/Dockerfile
 create mode 100755 db/configs/docker_generate_env.sh
 create mode 100755 db/migrate.sh
 create mode 100644 docker-compose.yml

diff --git a/README.md b/README.md
index 5fc88902..452b9151 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,28 @@ This monorepo contains the following components:
    [Bugout](https://bugout.dev). For more information on how that data is processed, check how the API
    inserts events from those sources into a stream.
 
+### Installation and setup
+
+#### Run server with Docker Compose
+
+If you want to deploy Moonstream in isolation against live services, then docker compose is your choice!
+
+- Run script `backend/configs/docker_generate_env.sh` which prepare for you:
+  - `backend/configs/docker.moonstreamapi.env` with environment variables
+- Run script `db/configs/docker_generate_env.sh` which prepare for you:
+  - `db/configs/alembic.moonstreamdb.ini` with postgresql uri
+
+```bash
+./backend/configs/docker_generate_env.sh
+./db/configs/docker_generate_env.sh
+```
+
+- Run local setup
+
+```bash
+docker-compose up --build
+```
+
 ## Contributing
 
 If you would like to contribute to Moonstream, please reach out to @zomglings on the [Moonstream Discord](https://discord.gg/pYE65FuNSz).
diff --git a/backend/Dockerfile b/backend/Dockerfile
new file mode 100644
index 00000000..64430872
--- /dev/null
+++ b/backend/Dockerfile
@@ -0,0 +1,20 @@
+FROM python:3.8-slim-buster
+
+# Update packages and
+# prepare alembic for docker compose setup
+RUN apt-get update && \
+    apt-get install -y libpq-dev gcc && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip3 install --no-cache-dir --upgrade pip setuptools && \
+    pip3 install --no-cache-dir psycopg2-binary alembic
+
+WORKDIR /usr/src/moonstreamapi
+
+COPY . /usr/src/moonstreamapi
+
+# Install Moonstream API package
+RUN pip3 install --no-cache-dir -e .
+
+EXPOSE 7481
+
+ENTRYPOINT ["./dev.sh"]
\ No newline at end of file
diff --git a/backend/configs/docker_generate_env.sh b/backend/configs/docker_generate_env.sh
new file mode 100755
index 00000000..d8de1910
--- /dev/null
+++ b/backend/configs/docker_generate_env.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env sh
+
+# Prepare Moonstream API application for docker-compose use
+
+set -e
+
+SCRIPT_DIR="$(realpath $(dirname $0))"
+DOCKER_MOONSTREAM_DB_URI="postgresql://postgres:postgres@db/moonstream_dev"
+DOCKER_MOONSTREAM_ENV_FILE="docker.moonstreamapi.env"
+
+# Generate environment variables
+
+cp "$SCRIPT_DIR/sample.env" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"
+
+# Clean file with variables from export prefix and quotation marks
+sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"
+sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"
+
+sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAM_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE"
diff --git a/backend/dev.sh b/backend/dev.sh
index 90b30c34..3080c352 100755
--- a/backend/dev.sh
+++ b/backend/dev.sh
@@ -1,9 +1,19 @@
 #!/usr/bin/env sh
 
-# Expects access to Python environment with the requirements for this project installed.
+# Sets up Moonstream API server
+# Expects access to Python environment with the requirements 
+# for this project installed.
 set -e
 
-MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}"
-MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}"
+MOONSTREAMAPI_HOST="${MOONSTREAMAPI_HOST:-127.0.0.1}"
+MOONSTREAMAPI_PORT="${MOONSTREAMAPI_PORT:-7481}"
+MOONSTREAMAPI_APP_DIR="${MOONSTREAMAPI_APP_DIR:-$PWD}"
+MOONSTREAMAPI_ASGI_APP="${MOONSTREAMAPI_ASGI_APP:-moonstreamapi.api:app}"
+MOONSTREAMAPI_UVICORN_WORKERS="${MOONSTREAMAPI_UVICORN_WORKERS:-2}"
 
-uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstreamapi.api:app --reload
+uvicorn --reload \
+  --port "$MOONSTREAMAPI_PORT" \
+  --host "$MOONSTREAMAPI_HOST" \
+  --app-dir "$MOONSTREAMAPI_APP_DIR" \
+  --workers "$MOONSTREAMAPI_UVICORN_WORKERS" \
+  "$MOONSTREAMAPI_ASGI_APP"
\ No newline at end of file
diff --git a/db/Dockerfile b/db/Dockerfile
new file mode 100644
index 00000000..df664b86
--- /dev/null
+++ b/db/Dockerfile
@@ -0,0 +1,18 @@
+FROM python:3.8-slim-buster
+
+# Update packages and
+# prepare alembic for docker compose setup
+RUN apt-get update && \
+    apt-get install -y libpq-dev gcc && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip3 install --no-cache-dir --upgrade pip setuptools && \
+    pip3 install --no-cache-dir psycopg2-binary alembic
+
+WORKDIR /usr/src/moonstreamdb
+
+COPY . /usr/src/moonstreamdb
+
+# Install Moonstream DB package
+RUN pip3 install --no-cache-dir -e .
+
+ENTRYPOINT ["./migrate.sh"]
\ No newline at end of file
diff --git a/db/configs/docker_generate_env.sh b/db/configs/docker_generate_env.sh
new file mode 100755
index 00000000..53ceed7e
--- /dev/null
+++ b/db/configs/docker_generate_env.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env sh
+
+# Prepare Moonstream DB application for docker-compose use
+
+set -e
+
+SCRIPT_DIR="$(realpath $(dirname $0))"
+DOCKER_MOONSTREAMDB_DB_URI="postgresql://postgres:postgres@db/moonstream_dev"
+DOCKER_MOONSTREAMDB_ENV_FILE="docker.moonstreamdb.env"
+DOCKER_MOONSTREAMDB_ALEMBIC_FILE="alembic.moonstreamdb.ini"
+
+# Generate environment variables
+
+cp "$SCRIPT_DIR/sample.env" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE"
+
+# Clean file with variables from export prefix and quotation marks
+sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE"
+sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE"
+
+sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE"
+
+# Generate alembic config
+
+cp "$SCRIPT_DIR/alembic.sample.ini" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ALEMBIC_FILE"
+
+sed -i "s|^sqlalchemy.url =.*|sqlalchemy.url = $DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ALEMBIC_FILE"
diff --git a/db/migrate.sh b/db/migrate.sh
new file mode 100755
index 00000000..5f8e573a
--- /dev/null
+++ b/db/migrate.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Sets up Brood server for docker compose:
+# 1. Running alembic migrations to head (using config file specified 
+# by the ALEMBIC_CONFIG environment variable)
+# 2. Running dev.sh (from the directory from which this script was called)
+
+set -e
+
+if [ -z "$ALEMBIC_CONFIG" ]
+then
+    echo "Please explicitly set the ALEMBIC_CONFIG environment variable to point to an alembic configuration file"
+    exit 1
+fi
+
+ALEMBIC_CONFIG="$ALEMBIC_CONFIG" sh alembic.sh upgrade head
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 00000000..614e353e
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,57 @@
+# Compose version
+version: "3"
+
+services:
+  # Moonstream API application
+  moonstreamapi:
+    build:
+      context: ./backend/
+      dockerfile: ./Dockerfile
+    image: moonstreamapi:latest
+    ports:
+      - "127.0.0.1:7481:7481"
+    # Specify environment file for compose setup
+    env_file: ./backend/configs/docker.moonstreamapi.env
+    environment:
+      MOONSTREAMAPI_HOST: 0.0.0.0
+      MOONSTREAMAPI_PORT: 7481
+      MOONSTREAMAPI_UVICORN_WORKERS: 1
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://moonstreamapi:7481/ping"]
+      interval: 5s
+      timeout: 1s
+      retries: 2
+      start_period: 2s
+    depends_on:
+      db:
+        condition: service_healthy
+  
+  # Moonstream DB application
+  moonstreamdb:
+    build:
+      context: ./db/
+      dockerfile: ./Dockerfile
+    image: moonstreamdb:latest
+    # Specify environment file for compose setup
+    env_file: ./db/configs/docker.moonstreamdb.env
+    environment:
+      ALEMBIC_CONFIG: ./configs/alembic.moonstreamdb.ini
+    depends_on:
+      db:
+        condition: service_healthy
+
+  # DB postgres application
+  db:
+    image: postgres:13
+    ports:
+      - "127.0.0.1:5432:5432"
+    environment:
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_DB: moonstream_dev
+    healthcheck:
+      test: ["CMD", "psql", "-U", "postgres", "-c", "SELECT 1;"]
+      interval: 5s
+      timeout: 1s
+      retries: 3
+      start_period: 2s