diff --git a/Dockerfiles/Dockerfile.stock_updater b/Dockerfiles/Dockerfile.stock_updater new file mode 100644 index 00000000..c68e87e3 --- /dev/null +++ b/Dockerfiles/Dockerfile.stock_updater @@ -0,0 +1,49 @@ +# syntax=docker/dockerfile:1 +FROM python:3.12-bookworm +LABEL authors="fureunoir" + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + LANG=C.UTF-8 \ + DEBIAN_FRONTEND=noninteractive \ + PATH="/root/.local/bin:$PATH" + +WORKDIR /app + +RUN set -eux; \ + sed -i 's|https://deb.debian.org/debian|https://ftp.uk.debian.org/debian|g' /etc/apt/sources.list.d/debian.sources; \ + apt-get update; \ + apt-get install -y --no-install-recommends wget gnupg; \ + wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -; \ + echo "deb http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends --fix-missing \ + build-essential \ + libpq-dev \ + gettext \ + libgettextpo-dev \ + graphviz-dev \ + libgts-dev \ + libpq5 \ + graphviz \ + binutils \ + libproj-dev \ + postgresql-client-17 \ + gdal-bin; \ + rm -rf /var/lib/apt/lists/*; \ + pip install --upgrade pip; \ + curl -sSL https://install.python-poetry.org | python3 + +COPY pyproject.toml pyproject.toml +COPY poetry.lock poetry.lock + +RUN poetry config virtualenvs.create false +RUN poetry install --extras="worker openai" --no-interaction --no-ansi + +COPY ./scripts/Docker/stock-updater-entrypoint.sh /usr/local/bin/stock-updater-entrypoint.sh +RUN chmod +x /usr/local/bin/stock-updater-entrypoint.sh + +COPY . . + +ENTRYPOINT ["stock-updater-entrypoint.sh"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f880c9ab..6396e4ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -149,6 +149,31 @@ services: retries: 5 start_period: 15s + stock_updater: + container_name: stock_updater + build: + context: . + dockerfile: ./Dockerfiles/Dockerfile.worker + restart: always + volumes: + - .:/app + env_file: + - .env + environment: + - BROKER_URL=${CELERY_BROKER_URL} + depends_on: + redis: + condition: service_healthy + elasticsearch: + condition: service_healthy + logging: *default-logging + healthcheck: + test: [ "CMD-SHELL", "celery -A evibes status | grep -q 'OK'" ] + interval: 30s + timeout: 10s + retries: 5 + start_period: 15s + beat: container_name: beat build: diff --git a/evibes/settings/celery.py b/evibes/settings/celery.py index 94f0c08b..e517d740 100644 --- a/evibes/settings/celery.py +++ b/evibes/settings/celery.py @@ -11,6 +11,7 @@ CELERY_BEAT_SCHEDULE = { "update_products_task": { "task": "core.tasks.update_products_task", "schedule": timedelta(minutes=60), + "options": {"queue": "stock_updater"}, }, "update_orderproducts_task": { "task": "core.tasks.update_orderproducts_task", diff --git a/scripts/Docker/stock-updater-entrypoint.sh b/scripts/Docker/stock-updater-entrypoint.sh new file mode 100644 index 00000000..1e5e868f --- /dev/null +++ b/scripts/Docker/stock-updater-entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/bash +set -e + +# wait for auxiliary services +poetry run python manage.py await_services + +# run stock_updater +poetry run celery -A evibes worker --pool=prefork --concurrency=1 --queues=stock_updater --loglevel=info --max-tasks-per-child=1 --pool-startup-timeout=60 \ No newline at end of file diff --git a/scripts/Docker/worker-entrypoint.sh b/scripts/Docker/worker-entrypoint.sh index 4d9541c8..9c5b26f6 100644 --- a/scripts/Docker/worker-entrypoint.sh +++ b/scripts/Docker/worker-entrypoint.sh @@ -4,5 +4,5 @@ set -e # wait for auxiliary services poetry run python manage.py await_services -# run workers and metrics exporter -poetry run celery -A evibes worker --loglevel=info -E --concurrency=4 --autoscale=4,2 --max-tasks-per-child=100 --max-memory-per-child=512000 --soft-time-limit=10800 --time-limit=21600 & /usr/local/bin/celery-prometheus-exporter \ No newline at end of file +# run worker and metrics exporter +poetry run celery -A evibes worker --pool=prefork --concurrency=8 --loglevel=info -E --queues=default --worker-prefetch-multiplier=1 --max-tasks-per-child=100 --max-memory-per-child=512000 --soft-time-limit=3600 --time-limit=7200 --pool-startup-timeout=30 & /usr/local/bin/celery-prometheus-exporter \ No newline at end of file