docker: restructure, separate production and development setup

.dockerignore

@@ -0,0 +1,115 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+jdav_web/db.sqlite3
+*.swp
+
+# test images for file upload
+#*.jpeg
+#*.png
+jdav_web/media
+jdav_web/*/migrations
+*.py.swp
+
+# django database migrations
+*/*/migrations/*
+/static/
+/media/
+
+# database configuration
+jdav_web/my.cnf
+
+# sass cache
+jdav_web/static/jet/css/themes/*/.sass-cache/*
+
+# test config
+config
+
+# celerybeat schedule database
+jdav_web/celerybeat-schedule.db

.gitignore

@@ -111,3 +111,6 @@ config
 
 # celerybeat schedule database
 jdav_web/celerybeat-schedule.db
+
+# docker environment variables
+docker.env

README.md

@@ -1,26 +1,19 @@
 # jdav_lb_webapp
 
-This repository has the purpose to develop a webapplication that can be used by 
+This repository has the purpose to develop a webapplication that can be used by
 JDAV to send newsletters, manage user lists and keep material lists up to date.
 As this repository is also meant to be a base for exchange during development, feel free
 to contribute ideas in form of edits to this README, issues, landmarks, projects, wiki entries, ...
 
-# Setup
+# Docker
 
-## Development server
+In the `docker` subfolder, there are `docker-compose.yaml`s for development and production use. For the development
+version, no further setup is needed.
 
-1. Install `virtualenv` (with pip3)
-2. Create a new virtualenv with `virtualenv --no-site-packages -p python3.9`
-3. Activate the environment with `source venv/bin/activate`
-4. Install requirements with `pip3 install -r requirements.txt`
-5. Setup a MySQL database called `jdav_db` and create a user that can access this database.
-6. Create a `my.cnf` at `jdav_web/my.cnf` with username and password for the mysql database.
-7. Change working directory to `jdav_web` and execute `python3 manage.py runserver`
+# Production
 
-## Deployed version
-
-- Mostly the same as the development version
-- Check the README on the server for further information.
+In production, the docker setup needs an external database. The exact access credentials are configured in the respective
+docker.env files.
 
 # Useful stuff
 

docker-compose.yaml

@@ -1,28 +0,0 @@
-version: "3.9"
-services:
-    nginx:
-        build: ./nginx/
-        restart: always
-        volumes:
-          - uwsgi_data:/tmp/uwsgi/
-          - web_static:/var/www/jdav_web/assets/:ro
-        ports:
-          - "8888:80"
-        depends_on:
-          - django
-
-    django:
-        build: ./jdav_web/
-        env_file: docker.env
-        restart: always
-        command: >
-          sh -c "python manage.py collectstatic --noinput
-          && uwsgi --ini kompass.uwsgi.ini"
-        volumes:
-          - uwsgi_data:/tmp/uwsgi/
-          - web_static:/code/static/
-          - web_static:/var/www/jdav_web/assets/
-
-volumes:
-  uwsgi_data:
-  web_static:

docker.env

@@ -1,18 +0,0 @@
-DJANGO_ALLOWED_HOST='*'
-DJANGO_BASE_URL='localhost:8008'
-DJANGO_PROTOCOL='http'
-
-EMAIL_HOST='hartley.uberspace.de'
-EMAIL_HOST_USER='post@flavigny.de'
-EMAIL_HOST_PASSWORD='Post00Flavigny..'
-EMAIL_SENDING_ADDRESS='post@flavigny.de'
-
-DJANGO_DEPLOY=1
-
-DJANGO_DATABASE_NAME='jdav_db'
-DJANGO_DATABASE_USER='jdav_db'
-DJANGO_DATABASE_PASSWORD='jdav00jdav'
-DJANGO_DATABASE_HOST='172.17.0.1'
-
-DJANGO_SETTINGS_MODULE='jdav_web.settings'
-DJANGO_STATIC_ROOT='/var/www/jdav_web/assets'

docker/development/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.9-bullseye
+
+# install additional dependencies
+RUN apt-get update && apt-get install -y gettext
+
+WORKDIR /app
+
+# install requirements
+COPY ./requirements.txt /app/requirements.txt
+RUN pip install -r requirements.txt
+
+COPY ./docker /app/docker

docker/development/docker-compose.yaml

@@ -0,0 +1,40 @@
+version: "3.9"
+
+x-kompass:
+    &kompass
+    image: kompass:development
+    build:
+        context: ./../../
+        dockerfile: docker/development/Dockerfile
+    env_file: docker.env
+    restart: always
+    depends_on:
+      - redis
+      - cache
+      - db
+
+services:
+    master:
+        <<: *kompass
+        entrypoint: /app/docker/development/entrypoint-master.sh
+        stdin_open: true # docker run -i
+        tty: true # docker run -t
+        volumes:
+          - ./../../jdav_web:/app/jdav_web
+        ports:
+          - "8000:8000"
+
+    cache:
+        restart: always
+        image: memcached:alpine
+
+    redis:
+        restart: always
+        image: redis:6-alpine
+
+    db:
+        restart: always
+        image: mariadb
+        volumes:
+          - /tmp/db:/var/lib/mysql
+        env_file: docker.env

docker/development/entrypoint-master.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+sleep 5
+
+cd /app
+
+if ! [ -f completed_initial_run ]; then
+    echo 'Initialising kompass master container'
+
+    python jdav_web/manage.py compilemessages --locale de
+
+    # python jdav_web/manage.py makemigrations
+    python jdav_web/manage.py migrate
+
+    touch completed_initial_run
+fi
+
+cd jdav_web
+
+celery -A jdav_web worker -B --scheduler django_celery_beat.schedulers:DatabaseScheduler -l info &
+python manage.py runserver 0.0.0.0:8000

docker/production/Dockerfile

@@ -0,0 +1,30 @@
+FROM python:3.9-bullseye
+
+# install additional dependencies
+RUN apt-get update && apt-get install -y gettext
+
+# create user
+RUN groupadd -g 501 app && useradd -g 501 -u 501 -m -d /app app
+
+# create static directory and set permissions, when doing this before mounting a named volume
+# in docker-compose.yaml, the permissions are inherited during the mount.
+RUN mkdir -p /var/www/jdav_web/assets && chown -R app:app /var/www/jdav_web/assets
+
+# create static directory and set permissions, when doing this before mounting a named volume
+# in docker-compose.yaml, the permissions are inherited during the mount.
+RUN mkdir -p /tmp/uwsgi && chown -R app:app /tmp/uwsgi
+
+WORKDIR /app
+USER app
+
+# add .local/bin to PATH
+ENV PATH="/app/.local/bin:$PATH"
+
+# install requirements
+COPY --chown=app:app ./requirements.txt /app/requirements.txt
+RUN pip install -r requirements.txt
+
+# for production, install uwsgi
+RUN pip install uwsgi
+
+COPY --chown=app:app . /app

docker/production/docker-compose.yaml

@@ -0,0 +1,53 @@
+version: "3.9"
+
+x-kompass:
+    &kompass
+    image: kompass:production
+    build:
+        context: ./../../
+        dockerfile: docker/production/Dockerfile
+    env_file: docker.env
+    restart: always
+    depends_on:
+      - redis
+      - cache
+
+services:
+    master:
+        <<: *kompass
+        entrypoint: /app/docker/production/entrypoint-master.sh
+        volumes:
+          - uwsgi_data:/tmp/uwsgi/
+          - web_static:/app/static/
+          - web_static:/var/www/jdav_web/assets/
+
+    nginx:
+        build: ./nginx/
+        restart: always
+        volumes:
+          - uwsgi_data:/tmp/uwsgi/
+          - web_static:/var/www/jdav_web/assets/:ro
+        ports:
+          - "3000:80"
+        depends_on:
+          - master
+
+    cache:
+        restart: always
+        image: memcached:alpine
+
+    redis:
+        restart: always
+        image: redis:6-alpine
+
+    celery_worker:
+        <<: *kompass
+        entrypoint: /app/docker/production/entrypoint-celery-worker.sh
+
+    celery_beat:
+        <<: *kompass
+        entrypoint: /app/docker/production/entrypoint-celery-beat.sh
+
+volumes:
+  uwsgi_data:
+  web_static:

docker/production/entrypoint-celery-beat.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cd /app/jdav_web
+
+celery -A jdav_web beat --scheduler django_celery_beat.schedulers:DatabaseScheduler -l info

docker/production/entrypoint-celery-worker.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cd /app/jdav_web
+
+celery -A jdav_web worker -l info

docker/production/entrypoint-master.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+cd /app
+
+if ! [ -f completed_initial_run ]; then
+    echo 'Initialising kompass master container'
+
+    python jdav_web/manage.py collectstatic --noinput
+    python jdav_web/manage.py compilemessages --locale de
+
+    python jdav_web/manage.py migrate
+
+    touch completed_initial_run
+fi
+
+uwsgi --ini docker/production/kompass.uwsgi.ini

docker/production/kompass.uwsgi.ini

@@ -1,6 +1,7 @@
 [uwsgi]
 
 socket = /tmp/uwsgi/kompass.sock
+chdir = /app/jdav_web/
 module = jdav_web.wsgi
 master = true
 processes = 2

jdav_web/Dockerfile

@@ -1,10 +0,0 @@
-FROM python:3.9-bullseye
-RUN mkdir /code
-WORKDIR /code
-COPY . /code
-
-# uwsgi setup
-RUN apt-get install libmariadb-dev
-RUN pip install uwsgi
-RUN pip install -r requirements.txt
-CMD ["uwsgi", "--ini", "/web/kompass.uwsgi.ini"]

jdav_web/jdav_web/settings.py

@@ -113,7 +113,7 @@ DATABASES = {
         'ENGINE': 'django.db.backends.mysql',
 	'NAME': os.environ.get('DJANGO_DATABASE_NAME', 'jdav_db'),
         'USER': os.environ.get('DJANGO_DATABASE_USER', 'jdav_user'),
-        'PASSWORD': os.environ.get('DJANGO_DATABASE_PASSWORD', 'password'),
+        'PASSWORD': os.environ.get('DJANGO_DATABASE_PASSWORD', 'jdav00jdav'),
         'HOST': os.environ.get('DJANGO_DATABASE_HOST', '127.0.0.1'),
         'PORT': os.environ.get('DJANGO_DATABASE_PORT', '5432')
     }
@@ -122,7 +122,7 @@ DATABASES = {
 CACHES = {
     'default': {
         'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
-        'LOCATION': '127.0.0.1:11211',
+        'LOCATION': os.environ.get('MEMCACHED_URL', '127.0.0.1:11211'),
     }
 }
 

requirements.txt

@@ -24,6 +24,7 @@ mysqlclient==2.1.0
 packaging==21.3
 Pillow==9.0.0
 prompt-toolkit==3.0.24
+pymemcache==4.0.0
 pyparsing==3.0.6
 python-crontab==2.7.1
 python-dateutil==2.8.2