diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1fcc753 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,115 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject +jdav_web/db.sqlite3 +*.swp + +# test images for file upload +#*.jpeg +#*.png +jdav_web/media +jdav_web/*/migrations +*.py.swp + +# django database migrations +*/*/migrations/* +/static/ +/media/ + +# database configuration +jdav_web/my.cnf + +# sass cache +jdav_web/static/jet/css/themes/*/.sass-cache/* + +# test config +config + +# celerybeat schedule database +jdav_web/celerybeat-schedule.db diff --git a/.gitignore b/.gitignore index 1ea8302..e4b0df2 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,6 @@ config # celerybeat schedule database jdav_web/celerybeat-schedule.db + +# docker environment variables +docker.env diff --git a/README.md b/README.md index 0b77c2c..da1f45f 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,19 @@ # jdav_lb_webapp -This repository has the purpose to develop a webapplication that can be used by +This repository has the purpose to develop a webapplication that can be used by JDAV to send newsletters, manage user lists and keep material lists up to date. As this repository is also meant to be a base for exchange during development, feel free to contribute ideas in form of edits to this README, issues, landmarks, projects, wiki entries, ... -# Setup +# Docker -## Development server +In the `docker` subfolder, there are `docker-compose.yaml`s for development and production use. For the development +version, no further setup is needed. -1. Install `virtualenv` (with pip3) -2. Create a new virtualenv with `virtualenv --no-site-packages -p python3.9` -3. Activate the environment with `source venv/bin/activate` -4. Install requirements with `pip3 install -r requirements.txt` -5. Setup a MySQL database called `jdav_db` and create a user that can access this database. -6. Create a `my.cnf` at `jdav_web/my.cnf` with username and password for the mysql database. -7. Change working directory to `jdav_web` and execute `python3 manage.py runserver` +# Production -## Deployed version - -- Mostly the same as the development version -- Check the README on the server for further information. +In production, the docker setup needs an external database. The exact access credentials are configured in the respective +docker.env files. # Useful stuff diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index e60f32e..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,28 +0,0 @@ -version: "3.9" -services: - nginx: - build: ./nginx/ - restart: always - volumes: - - uwsgi_data:/tmp/uwsgi/ - - web_static:/var/www/jdav_web/assets/:ro - ports: - - "8888:80" - depends_on: - - django - - django: - build: ./jdav_web/ - env_file: docker.env - restart: always - command: > - sh -c "python manage.py collectstatic --noinput - && uwsgi --ini kompass.uwsgi.ini" - volumes: - - uwsgi_data:/tmp/uwsgi/ - - web_static:/code/static/ - - web_static:/var/www/jdav_web/assets/ - -volumes: - uwsgi_data: - web_static: diff --git a/docker.env b/docker.env deleted file mode 100644 index ef2154f..0000000 --- a/docker.env +++ /dev/null @@ -1,18 +0,0 @@ -DJANGO_ALLOWED_HOST='*' -DJANGO_BASE_URL='localhost:8008' -DJANGO_PROTOCOL='http' - -EMAIL_HOST='hartley.uberspace.de' -EMAIL_HOST_USER='post@flavigny.de' -EMAIL_HOST_PASSWORD='Post00Flavigny..' -EMAIL_SENDING_ADDRESS='post@flavigny.de' - -DJANGO_DEPLOY=1 - -DJANGO_DATABASE_NAME='jdav_db' -DJANGO_DATABASE_USER='jdav_db' -DJANGO_DATABASE_PASSWORD='jdav00jdav' -DJANGO_DATABASE_HOST='172.17.0.1' - -DJANGO_SETTINGS_MODULE='jdav_web.settings' -DJANGO_STATIC_ROOT='/var/www/jdav_web/assets' diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile new file mode 100644 index 0000000..a0a9ab4 --- /dev/null +++ b/docker/development/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.9-bullseye + +# install additional dependencies +RUN apt-get update && apt-get install -y gettext + +WORKDIR /app + +# install requirements +COPY ./requirements.txt /app/requirements.txt +RUN pip install -r requirements.txt + +COPY ./docker /app/docker diff --git a/docker/development/docker-compose.yaml b/docker/development/docker-compose.yaml new file mode 100644 index 0000000..e1520a1 --- /dev/null +++ b/docker/development/docker-compose.yaml @@ -0,0 +1,40 @@ +version: "3.9" + +x-kompass: + &kompass + image: kompass:development + build: + context: ./../../ + dockerfile: docker/development/Dockerfile + env_file: docker.env + restart: always + depends_on: + - redis + - cache + - db + +services: + master: + <<: *kompass + entrypoint: /app/docker/development/entrypoint-master.sh + stdin_open: true # docker run -i + tty: true # docker run -t + volumes: + - ./../../jdav_web:/app/jdav_web + ports: + - "8000:8000" + + cache: + restart: always + image: memcached:alpine + + redis: + restart: always + image: redis:6-alpine + + db: + restart: always + image: mariadb + volumes: + - /tmp/db:/var/lib/mysql + env_file: docker.env diff --git a/docker/development/entrypoint-master.sh b/docker/development/entrypoint-master.sh new file mode 100755 index 0000000..5d431ff --- /dev/null +++ b/docker/development/entrypoint-master.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -o errexit + +sleep 5 + +cd /app + +if ! [ -f completed_initial_run ]; then + echo 'Initialising kompass master container' + + python jdav_web/manage.py compilemessages --locale de + + # python jdav_web/manage.py makemigrations + python jdav_web/manage.py migrate + + touch completed_initial_run +fi + +cd jdav_web + +celery -A jdav_web worker -B --scheduler django_celery_beat.schedulers:DatabaseScheduler -l info & +python manage.py runserver 0.0.0.0:8000 diff --git a/docker/production/Dockerfile b/docker/production/Dockerfile new file mode 100644 index 0000000..e063e70 --- /dev/null +++ b/docker/production/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.9-bullseye + +# install additional dependencies +RUN apt-get update && apt-get install -y gettext + +# create user +RUN groupadd -g 501 app && useradd -g 501 -u 501 -m -d /app app + +# create static directory and set permissions, when doing this before mounting a named volume +# in docker-compose.yaml, the permissions are inherited during the mount. +RUN mkdir -p /var/www/jdav_web/assets && chown -R app:app /var/www/jdav_web/assets + +# create static directory and set permissions, when doing this before mounting a named volume +# in docker-compose.yaml, the permissions are inherited during the mount. +RUN mkdir -p /tmp/uwsgi && chown -R app:app /tmp/uwsgi + +WORKDIR /app +USER app + +# add .local/bin to PATH +ENV PATH="/app/.local/bin:$PATH" + +# install requirements +COPY --chown=app:app ./requirements.txt /app/requirements.txt +RUN pip install -r requirements.txt + +# for production, install uwsgi +RUN pip install uwsgi + +COPY --chown=app:app . /app diff --git a/docker/production/docker-compose.yaml b/docker/production/docker-compose.yaml new file mode 100644 index 0000000..266c3e5 --- /dev/null +++ b/docker/production/docker-compose.yaml @@ -0,0 +1,53 @@ +version: "3.9" + +x-kompass: + &kompass + image: kompass:production + build: + context: ./../../ + dockerfile: docker/production/Dockerfile + env_file: docker.env + restart: always + depends_on: + - redis + - cache + +services: + master: + <<: *kompass + entrypoint: /app/docker/production/entrypoint-master.sh + volumes: + - uwsgi_data:/tmp/uwsgi/ + - web_static:/app/static/ + - web_static:/var/www/jdav_web/assets/ + + nginx: + build: ./nginx/ + restart: always + volumes: + - uwsgi_data:/tmp/uwsgi/ + - web_static:/var/www/jdav_web/assets/:ro + ports: + - "3000:80" + depends_on: + - master + + cache: + restart: always + image: memcached:alpine + + redis: + restart: always + image: redis:6-alpine + + celery_worker: + <<: *kompass + entrypoint: /app/docker/production/entrypoint-celery-worker.sh + + celery_beat: + <<: *kompass + entrypoint: /app/docker/production/entrypoint-celery-beat.sh + +volumes: + uwsgi_data: + web_static: diff --git a/docker/production/entrypoint-celery-beat.sh b/docker/production/entrypoint-celery-beat.sh new file mode 100755 index 0000000..f63751e --- /dev/null +++ b/docker/production/entrypoint-celery-beat.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +cd /app/jdav_web + +celery -A jdav_web beat --scheduler django_celery_beat.schedulers:DatabaseScheduler -l info diff --git a/docker/production/entrypoint-celery-worker.sh b/docker/production/entrypoint-celery-worker.sh new file mode 100755 index 0000000..9cc09b1 --- /dev/null +++ b/docker/production/entrypoint-celery-worker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +cd /app/jdav_web + +celery -A jdav_web worker -l info diff --git a/docker/production/entrypoint-master.sh b/docker/production/entrypoint-master.sh new file mode 100755 index 0000000..d07446a --- /dev/null +++ b/docker/production/entrypoint-master.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -o errexit + +cd /app + +if ! [ -f completed_initial_run ]; then + echo 'Initialising kompass master container' + + python jdav_web/manage.py collectstatic --noinput + python jdav_web/manage.py compilemessages --locale de + + python jdav_web/manage.py migrate + + touch completed_initial_run +fi + +uwsgi --ini docker/production/kompass.uwsgi.ini diff --git a/jdav_web/kompass.uwsgi.ini b/docker/production/kompass.uwsgi.ini similarity index 84% rename from jdav_web/kompass.uwsgi.ini rename to docker/production/kompass.uwsgi.ini index f1a7451..8566eab 100644 --- a/jdav_web/kompass.uwsgi.ini +++ b/docker/production/kompass.uwsgi.ini @@ -1,6 +1,7 @@ [uwsgi] socket = /tmp/uwsgi/kompass.sock +chdir = /app/jdav_web/ module = jdav_web.wsgi master = true processes = 2 diff --git a/nginx/Dockerfile b/docker/production/nginx/Dockerfile similarity index 100% rename from nginx/Dockerfile rename to docker/production/nginx/Dockerfile diff --git a/nginx/kompass.nginx.conf b/docker/production/nginx/kompass.nginx.conf similarity index 100% rename from nginx/kompass.nginx.conf rename to docker/production/nginx/kompass.nginx.conf diff --git a/nginx/nginx.conf b/docker/production/nginx/nginx.conf similarity index 100% rename from nginx/nginx.conf rename to docker/production/nginx/nginx.conf diff --git a/jdav_web/Dockerfile b/jdav_web/Dockerfile deleted file mode 100644 index dd07ac2..0000000 --- a/jdav_web/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.9-bullseye -RUN mkdir /code -WORKDIR /code -COPY . /code - -# uwsgi setup -RUN apt-get install libmariadb-dev -RUN pip install uwsgi -RUN pip install -r requirements.txt -CMD ["uwsgi", "--ini", "/web/kompass.uwsgi.ini"] diff --git a/jdav_web/jdav_web/settings.py b/jdav_web/jdav_web/settings.py index 32d8159..ac4804a 100644 --- a/jdav_web/jdav_web/settings.py +++ b/jdav_web/jdav_web/settings.py @@ -113,7 +113,7 @@ DATABASES = { 'ENGINE': 'django.db.backends.mysql', 'NAME': os.environ.get('DJANGO_DATABASE_NAME', 'jdav_db'), 'USER': os.environ.get('DJANGO_DATABASE_USER', 'jdav_user'), - 'PASSWORD': os.environ.get('DJANGO_DATABASE_PASSWORD', 'password'), + 'PASSWORD': os.environ.get('DJANGO_DATABASE_PASSWORD', 'jdav00jdav'), 'HOST': os.environ.get('DJANGO_DATABASE_HOST', '127.0.0.1'), 'PORT': os.environ.get('DJANGO_DATABASE_PORT', '5432') } @@ -122,7 +122,7 @@ DATABASES = { CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'LOCATION': '127.0.0.1:11211', + 'LOCATION': os.environ.get('MEMCACHED_URL', '127.0.0.1:11211'), } } diff --git a/jdav_web/requirements.txt b/requirements.txt similarity index 97% rename from jdav_web/requirements.txt rename to requirements.txt index 1afdf87..d482b6c 100644 --- a/jdav_web/requirements.txt +++ b/requirements.txt @@ -24,6 +24,7 @@ mysqlclient==2.1.0 packaging==21.3 Pillow==9.0.0 prompt-toolkit==3.0.24 +pymemcache==4.0.0 pyparsing==3.0.6 python-crontab==2.7.1 python-dateutil==2.8.2