Update Celery settings and worker configuration

Refactored Celery configurations to enhance task reliability, including changes to prefetch multiplier and timeout settings. Updated the worker command in `docker-compose.yml` to specify concurrency, memory limits, and task timeouts. Added a memory limit for the worker service to improve resource management.

docker-compose.yml

@@ -43,7 +43,15 @@ services:
     container_name: worker
     build: .
     restart: always
-    command: sh -c "poetry run celery -A evibes worker --loglevel=info --autoscale=10,3"
+    command: >
+      sh -c "poetry run celery -A evibes worker
+        --loglevel=info
+        --concurrency=4
+        --autoscale=4,2
+        --max-tasks-per-child=100
+        --max-memory-per-child=512000
+        --soft-time-limit=1800
+        --time-limit=3600"
     volumes:
       - .:/app
     env_file:
@@ -57,6 +65,7 @@ services:
       interval: 30s
       timeout: 10s
       retries: 5
+    mem_limit: 2g
 
   beat:
     container_name: beat

evibes/celery.py

@@ -15,8 +15,6 @@ app.conf.update(
     worker_log_format="[%(asctime)s: %(levelname)s/%(processName)s] %(name)s: %(message)s",
     worker_task_log_format="[%(asctime)s: %(levelname)s/%(processName)s] %(name)s: %(message)s",
     worker_autoscale=(10, 3),
-    worker_prefetch_multiplier=1,
-    worker_max_tasks_per_child=100,
     broker_connection_retry_on_startup=True,
     timezone=TIME_ZONE,
     task_serializer="json",
@@ -27,6 +25,12 @@ app.conf.update(
         "retry_policy": {"interval_start": 0.1, "interval_step": 0.2, "max_retries": 5},
         "visibility_timeout": 3600,
     },
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    worker_prefetch_multiplier=1,
+    worker_max_tasks_per_child=100,
+    task_soft_time_limit=1800,
+    task_time_limit=3600,
 )
 
 app.config_from_object("django.conf:settings", namespace="CELERY")