diff --git a/docker-compose.yml b/docker-compose.yml index 906a12d5..17d74b5f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,15 @@ services: container_name: worker build: . restart: always - command: sh -c "poetry run celery -A evibes worker --loglevel=info --autoscale=10,3" + command: > + sh -c "poetry run celery -A evibes worker + --loglevel=info + --concurrency=4 + --autoscale=4,2 + --max-tasks-per-child=100 + --max-memory-per-child=512000 + --soft-time-limit=1800 + --time-limit=3600" volumes: - .:/app env_file: @@ -57,6 +65,7 @@ services: interval: 30s timeout: 10s retries: 5 + mem_limit: 2g beat: container_name: beat diff --git a/evibes/celery.py b/evibes/celery.py index 722ee37f..da158a25 100644 --- a/evibes/celery.py +++ b/evibes/celery.py @@ -15,8 +15,6 @@ app.conf.update( worker_log_format="[%(asctime)s: %(levelname)s/%(processName)s] %(name)s: %(message)s", worker_task_log_format="[%(asctime)s: %(levelname)s/%(processName)s] %(name)s: %(message)s", worker_autoscale=(10, 3), - worker_prefetch_multiplier=1, - worker_max_tasks_per_child=100, broker_connection_retry_on_startup=True, timezone=TIME_ZONE, task_serializer="json", @@ -27,6 +25,12 @@ app.conf.update( "retry_policy": {"interval_start": 0.1, "interval_step": 0.2, "max_retries": 5}, "visibility_timeout": 3600, }, + task_acks_late=True, + task_reject_on_worker_lost=True, + worker_prefetch_multiplier=1, + worker_max_tasks_per_child=100, + task_soft_time_limit=1800, + task_time_limit=3600, ) app.config_from_object("django.conf:settings", namespace="CELERY")