Merge pull request #4 from bugout-dev/backend-scratch

Moonstream backend init

backend/.gitignore

@@ -0,0 +1,167 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode
+
+# Custom
+dev.env
+prod.env
+.moonstream
+.env

backend/deploy/deploy.bash

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Deployment script - intended to run on Moonstream servers
+
+# Main
+APP_DIR="${APP_DIR:-/home/ubuntu/app}"
+AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
+PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/app-env}"
+PYTHON="${PYTHON_ENV_DIR}/bin/python"
+PIP="${PYTHON_ENV_DIR}/bin/pip"
+SCRIPT_DIR="$(realpath $(dirname $0))"
+PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
+SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/app-secrets}"
+PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
+AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
+SERVICE_FILE="${SCRIPT_DIR}/moonstream.service"
+
+set -eu
+
+echo
+echo
+echo "Updating Python dependencies"
+"${PIP}" install -r "${APP_DIR}/requirements.txt"
+
+echo
+echo
+echo "Retrieving deployment parameters"
+mkdir -p "${SECRETS_DIR}"
+AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "${AWS_SSM_PARAMETER_PATH}" -o "${PARAMETERS_ENV_PATH}"
+
+echo
+echo
+echo "Replacing existing Moonstream service definition with ${SERVICE_FILE}"
+chmod 644 "${SERVICE_FILE}"
+cp "${SERVICE_FILE}" /etc/systemd/system/moonstream.service
+systemctl daemon-reload
+systemctl restart moonstream.service
+systemctl status moonstream.service

backend/deploy/moonstream.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=moonstream-service
+After=network.target
+
+[Service]
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/app
+EnvironmentFile=/home/ubuntu/secrets/app.env
+ExecStart=/home/ubuntu/server-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app
+SyslogIdentifier=moonstream
+
+[Install]
+WantedBy=multi-user.target

backend/deploy/parameters.py

@@ -0,0 +1,86 @@
+"""
+Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
+"""
+import argparse
+from dataclasses import dataclass
+import sys
+from typing import Any, Dict, Iterable, List, Optional
+
+import boto3
+
+
+@dataclass
+class EnvironmentVariable:
+    name: str
+    value: str
+
+
+def get_parameters(path: str) -> List[Dict[str, Any]]:
+    """
+    Retrieve parameters from AWS SSM Parameter Store. Decrypts any encrypted parameters.
+
+    Relies on the appropriate environment variables to authenticate against AWS:
+    https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
+    """
+    ssm = boto3.client("ssm")
+    next_token: Optional[bool] = True
+    parameters: List[Dict[str, Any]] = []
+    while next_token is not None:
+        kwargs = {"Path": path, "Recursive": False, "WithDecryption": True}
+        if next_token is not True:
+            kwargs["NextToken"] = next_token
+        response = ssm.get_parameters_by_path(**kwargs)
+        new_parameters = response.get("Parameters", [])
+        parameters.extend(new_parameters)
+        next_token = response.get("NextToken")
+
+    return parameters
+
+
+def parameter_to_env(parameter_object: Dict[str, Any]) -> EnvironmentVariable:
+    """
+    Transforms parameters returned by the AWS SSM API into EnvironmentVariables.
+    """
+    parameter_path = parameter_object.get("Name")
+    if parameter_path is None:
+        raise ValueError('Did not find "Name" in parameter object')
+    name = parameter_path.split("/")[-1].upper()
+
+    value = parameter_object.get("Value")
+    if value is None:
+        raise ValueError('Did not find "Value" in parameter object')
+
+    return EnvironmentVariable(name, value)
+
+
+def env_string(env_vars: Iterable[EnvironmentVariable], with_export: bool) -> str:
+    """
+    Produces a string which, when executed in a shell, exports the desired environment variables as
+    specified by env_vars.
+    """
+    prefix = "export " if with_export else ""
+    return "\n".join([f'{prefix}{var.name}="{var.value}"' for var in env_vars])
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Materialize environment variables from AWS SSM Parameter Store"
+    )
+    parser.add_argument(
+        "path",
+        help="SSM path from which to pull environment variables (pull is NOT recursive)",
+    )
+    parser.add_argument(
+        "-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout
+    )
+    parser.add_argument(
+        "--export",
+        action="store_true",
+        help="Set to output environment strings with export statements",
+    )
+
+    args = parser.parse_args()
+
+    result = env_string(map(parameter_to_env, get_parameters(args.path)), args.export)
+    with args.outfile as ofp:
+        print(result, file=ofp)

backend/dev.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+# Expects access to Python environment with the requirements for this project installed.
+set -e
+
+MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}"
+MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}"
+
+uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstream.api:app --workers 2 $@

backend/moonstream/README.md

@@ -0,0 +1 @@
+# moonstream backend

backend/moonstream/api.py

@@ -0,0 +1,58 @@
+"""
+The Moonstream HTTP API
+"""
+import logging
+from typing import Any, Dict, List, Optional
+import uuid
+
+from fastapi import (
+    BackgroundTasks,
+    Depends,
+    FastAPI,
+    Form,
+    HTTPException,
+    Path,
+    Query,
+    Request,
+    Response,
+)
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.security import OAuth2PasswordRequestForm
+
+from . import data
+from .settings import DOCS_TARGET_PATH, ORIGINS
+from .version import MOONSTREAM_VERSION
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+tags_metadata = [{"name": "users", "description": "Operations with users."}]
+
+app = FastAPI(
+    title=f"Moonstream API.",
+    description="The Bugout blockchain inspector API.",
+    version=MOONSTREAM_VERSION,
+    openapi_tags=tags_metadata,
+    openapi_url="/openapi.json",
+    docs_url=None,
+    redoc_url=f"/{DOCS_TARGET_PATH}",
+)
+
+# CORS settings
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/ping", response_model=data.PingResponse)
+async def ping() -> data.PingResponse:
+    return data.PingResponse(status="ok")
+
+
+@app.get("/version", response_model=data.VersionResponse)
+async def version() -> data.VersionResponse:
+    return data.VersionResponse(version=MOONSTREAM_VERSION)

backend/moonstream/data.py

@@ -0,0 +1,20 @@
+"""
+Pydantic schemas for the Moonstream HTTP API
+"""
+from pydantic import BaseModel
+
+
+class PingResponse(BaseModel):
+    """
+    Schema for ping response
+    """
+
+    status: str
+
+
+class VersionResponse(BaseModel):
+    """
+    Schema for responses on /version endpoint
+    """
+
+    version: str

backend/moonstream/requirements.txt

@@ -0,0 +1,29 @@
+appdirs==1.4.4
+asgiref==3.4.1
+black==21.7b0
+boto3==1.18.1
+botocore==1.21.1
+bugout==0.1.12
+certifi==2021.5.30
+charset-normalizer==2.0.3
+click==8.0.1
+fastapi==0.66.0
+h11==0.12.0
+idna==3.2
+jmespath==0.10.0
+mypy==0.910
+mypy-extensions==0.4.3
+pathspec==0.9.0
+pkg-resources==0.0.0
+pydantic==1.8.2
+python-dateutil==2.8.2
+regex==2021.7.6
+requests==2.26.0
+s3transfer==0.5.0
+six==1.16.0
+starlette==0.14.2
+toml==0.10.2
+tomli==1.0.4
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+uvicorn==0.14.0

backend/moonstream/settings.py

@@ -0,0 +1,16 @@
+import os
+
+# Origin
+RAW_ORIGIN = os.environ.get("MOONSTREAM_CORS_ALLOWED_ORIGINS")
+if RAW_ORIGIN is None:
+    raise ValueError(
+        "MOONSTREAM_CORS_ALLOWED_ORIGINS environment variable must be set (comma-separated list of CORS allowed origins"
+    )
+ORIGINS = RAW_ORIGIN.split(",")
+
+# OpenAPI
+DOCS_TARGET_PATH = "docs"
+MOONSTREAM_OPENAPI_LIST = []
+MOONSTREAM_OPENAPI_LIST_RAW = os.environ.get("MOONSTREAM_OPENAPI_LIST")
+if MOONSTREAM_OPENAPI_LIST_RAW is not None:
+    MOONSTREAM_OPENAPI_LIST = MOONSTREAM_OPENAPI_LIST_RAW.split(",")

backend/moonstream/version.py

@@ -0,0 +1,5 @@
+"""
+Moonstream library and API version.
+"""
+
+MOONSTREAM_VERSION = "0.0.1"

backend/sample.env

@@ -0,0 +1,2 @@
+export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to"
+export MOONSTREAM_OPENAPI_LIST=""

backend/setup.py

@@ -0,0 +1,33 @@
+from setuptools import find_packages, setup
+
+from moonstream.version import MOONSTREAM_VERSION
+
+long_description = ""
+with open("README.md") as ifp:
+    long_description = ifp.read()
+
+setup(
+    name="moonstream",
+    version=MOONSTREAM_VERSION,
+    packages=find_packages(),
+    install_requires=["boto3", "bugout", "fastapi", "uvicorn"],
+    extras_require={
+        "dev": ["black", "mypy"],
+        "distribute": ["setuptools", "twine", "wheel"],
+    },
+    package_data={"moonstream": ["py.typed"]},
+    zip_safe=False,
+    description="The Bugout blockchain inspector API.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    author="Bugout.dev",
+    author_email="engineering@bugout.dev",
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Programming Language :: Python",
+        "License :: OSI Approved :: Apache Software License",
+        "Topic :: Software Development :: Libraries",
+    ],
+    url="https://github.com/bugout-dev/moonstream",
+    entry_points={"console_scripts": ["moonstream=moonstream.cli:main"]},
+)