Features: 1) Add stock_updater service to docker-compose.yml with health checks and logging; 2) Introduce Dockerfile.stock_updater for stock_updater service with dependencies and entrypoint; 3) Add stock-updater-entrypoint.sh script for Celery worker initialization; 4) Specify stock_updater queue in Celery task configuration;

Fixes: 1) Update Celery worker configuration in `worker-entrypoint.sh` to optimize worker pool and task limits;

Extra: Refactor `docker-compose.yml` and scripts for improved service management.

Dockerfiles/Dockerfile.stock_updater

@@ -0,0 +1,49 @@
+# syntax=docker/dockerfile:1
+FROM python:3.12-bookworm
+LABEL authors="fureunoir"
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    PATH="/root/.local/bin:$PATH"
+
+WORKDIR /app
+
+RUN set -eux; \
+    sed -i 's|https://deb.debian.org/debian|https://ftp.uk.debian.org/debian|g' /etc/apt/sources.list.d/debian.sources; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends wget gnupg; \
+    wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -; \
+    echo "deb http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" \
+         > /etc/apt/sources.list.d/pgdg.list; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends --fix-missing \
+        build-essential \
+        libpq-dev \
+        gettext \
+        libgettextpo-dev \
+        graphviz-dev \
+        libgts-dev \
+        libpq5 \
+        graphviz \
+        binutils \
+        libproj-dev \
+        postgresql-client-17 \
+        gdal-bin; \
+    rm -rf /var/lib/apt/lists/*; \
+    pip install --upgrade pip; \
+    curl -sSL https://install.python-poetry.org | python3
+
+COPY pyproject.toml pyproject.toml
+COPY poetry.lock poetry.lock
+
+RUN poetry config virtualenvs.create false
+RUN poetry install --extras="worker openai" --no-interaction --no-ansi
+
+COPY ./scripts/Docker/stock-updater-entrypoint.sh /usr/local/bin/stock-updater-entrypoint.sh
+RUN chmod +x /usr/local/bin/stock-updater-entrypoint.sh
+
+COPY . .
+
+ENTRYPOINT ["stock-updater-entrypoint.sh"]

docker-compose.yml

@@ -149,6 +149,31 @@ services:
       retries: 5
       start_period: 15s
 
+  stock_updater:
+    container_name: stock_updater
+    build:
+      context: .
+      dockerfile: ./Dockerfiles/Dockerfile.worker
+    restart: always
+    volumes:
+      - .:/app
+    env_file:
+      - .env
+    environment:
+      - BROKER_URL=${CELERY_BROKER_URL}
+    depends_on:
+      redis:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_healthy
+    logging: *default-logging
+    healthcheck:
+      test: [ "CMD-SHELL", "celery -A evibes status | grep -q 'OK'" ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 15s
+
   beat:
     container_name: beat
     build:

evibes/settings/celery.py

@@ -11,6 +11,7 @@ CELERY_BEAT_SCHEDULE = {
     "update_products_task": {
         "task": "core.tasks.update_products_task",
         "schedule": timedelta(minutes=60),
+        "options": {"queue": "stock_updater"},
     },
     "update_orderproducts_task": {
         "task": "core.tasks.update_orderproducts_task",

scripts/Docker/stock-updater-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+set -e
+
+# wait for auxiliary services
+poetry run python manage.py await_services
+
+# run stock_updater
+poetry run celery -A evibes worker --pool=prefork --concurrency=1 --queues=stock_updater --loglevel=info --max-tasks-per-child=1 --pool-startup-timeout=60

scripts/Docker/worker-entrypoint.sh

@@ -4,5 +4,5 @@ set -e
 # wait for auxiliary services
 poetry run python manage.py await_services
 
-# run workers and metrics exporter
+# run worker and metrics exporter
-poetry run celery -A evibes worker --loglevel=info -E --concurrency=4 --autoscale=4,2 --max-tasks-per-child=100 --max-memory-per-child=512000 --soft-time-limit=10800 --time-limit=21600 & /usr/local/bin/celery-prometheus-exporter
+poetry run celery -A evibes worker --pool=prefork --concurrency=8 --loglevel=info -E --queues=default --worker-prefetch-multiplier=1 --max-tasks-per-child=100 --max-memory-per-child=512000 --soft-time-limit=3600 --time-limit=7200 --pool-startup-timeout=30 & /usr/local/bin/celery-prometheus-exporter