Files

205 lines
7.6 KiB
Python

"""
Moodle Queue Service (admin-edu-space)
Provides asynchronous, fault-tolerant queuing and processing of synchronization
operations between admin-edu-space and Moodle 4.1.
Implements Exponential Backoff and Dead Letter Queue (DLQ).
"""
import logging
from datetime import datetime, timedelta, timezone
from typing import Optional
from app import db
from app.models.sync_task import MoodleSyncTask
from app.services.moodle_client import moodle_client
logger = logging.getLogger(__name__)
class MoodleQueueService:
@staticmethod
def enqueue_task(action: str, entity_type: str, entity_id: Optional[str] = None, payload: Optional[dict] = None, max_attempts: int = 5) -> MoodleSyncTask:
"""
Enqueues a new synchronization task to be processed asynchronously.
Guarantees that local transactions are never blocked by Moodle unavailability.
"""
task = MoodleSyncTask(
action=action,
entity_type=entity_type,
entity_id=entity_id if entity_id is not None else None,
payload=payload or {},
status='PENDING',
attempts=0,
max_attempts=max_attempts,
next_retry_at=datetime.now(timezone.utc).replace(tzinfo=None)
)
db.session.add(task)
db.session.commit()
logger.info(f"[MoodleQueue] Enqueued task {task.id}: action={action}, entity={entity_type}:{entity_id}")
return task
@staticmethod
def process_pending_tasks(batch_size: int = 20) -> dict:
"""
Processes a batch of pending or retrying tasks.
Uses exponential backoff for retries and sends to Dead Letter Queue (FAILED)
if max_attempts are exceeded.
"""
now = datetime.now(timezone.utc).replace(tzinfo=None)
tasks = MoodleSyncTask.query.filter(
MoodleSyncTask.status.in_(['PENDING', 'RETRYING']),
(MoodleSyncTask.next_retry_at == None) | (MoodleSyncTask.next_retry_at <= now)
).order_by(MoodleSyncTask.created_at.asc()).limit(batch_size).all()
results = {
'processed': 0,
'succeeded': 0,
'failed': 0,
'retrying': 0
}
if not tasks:
return results
for task in tasks:
results['processed'] += 1
task.status = 'PROCESSING'
task.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit()
try:
MoodleQueueService._execute_task_action(task)
task.status = 'COMPLETED'
task.error_message = None
task.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit()
results['succeeded'] += 1
logger.info(f"[MoodleQueue] Task {task.id} ({task.action}) completed successfully.")
except Exception as e:
task.attempts += 1
task.error_message = str(e)
task.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
if task.attempts >= task.max_attempts:
task.status = 'FAILED' # Dead Letter Queue (DLQ)
task.next_retry_at = None
results['failed'] += 1
logger.error(f"[MoodleQueue] Task {task.id} permanently failed (DLQ): {e}")
else:
task.status = 'RETRYING'
# Exponential Backoff: 30s, 60s, 120s, 240s... capped at 1 hour
delay_seconds = min(3600, (2 ** task.attempts) * 30)
task.next_retry_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(seconds=delay_seconds)
results['retrying'] += 1
logger.warning(f"[MoodleQueue] Task {task.id} failed attempt {task.attempts}/{task.max_attempts}. Next retry in {delay_seconds}s: {e}")
db.session.commit()
return results
@staticmethod
def _execute_task_action(task: MoodleSyncTask):
"""
Executes the specific Moodle Web Service operation.
Raises an exception on failure or error response.
"""
payload = task.payload or {}
action = task.action.upper()
if action == 'CREATE_USER':
users = payload.get('users') or [payload]
res = moodle_client.create_users(users)
return res
elif action == 'UPDATE_USER':
users = payload.get('users') or [payload]
res = moodle_client.update_users(users)
return res
elif action == 'ENROL_USER':
enrolments = payload.get('enrolments') or [payload]
res = moodle_client.enrol_users(enrolments)
return res
elif action == 'UNENROL_USER':
enrolments = payload.get('enrolments') or [payload]
res = moodle_client.unenrol_users(enrolments)
return res
elif action == 'ASSIGN_ROLE':
role_id = int(payload.get('role_id', 5))
user_id = int(payload.get('user_id', 0))
context_id = int(payload.get('context_id', 1))
res = moodle_client.assign_role(role_id, user_id, context_id)
return res
elif action == 'UNASSIGN_ROLE':
role_id = int(payload.get('role_id', 5))
user_id = int(payload.get('user_id', 0))
context_id = int(payload.get('context_id', 1))
res = moodle_client.unassign_role(role_id, user_id, context_id)
return res
else:
raise ValueError(f"Unsupported sync action: {action}")
@staticmethod
def retry_task(task_id: int) -> bool:
"""
Manually re-enqueues a task from DLQ or error state back to PENDING.
"""
task = MoodleSyncTask.query.get(task_id)
if not task:
return False
task.status = 'PENDING'
task.attempts = 0
task.next_retry_at = datetime.now(timezone.utc).replace(tzinfo=None)
task.error_message = None
task.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
db.session.commit()
logger.info(f"[MoodleQueue] Task {task_id} manually reset to PENDING.")
return True
@staticmethod
def retry_all_failed() -> int:
"""
Retries all tasks in FAILED status (DLQ).
"""
failed_tasks = MoodleSyncTask.query.filter_by(status='FAILED').all()
count = 0
for task in failed_tasks:
task.status = 'PENDING'
task.attempts = 0
task.next_retry_at = datetime.now(timezone.utc).replace(tzinfo=None)
task.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
count += 1
db.session.commit()
logger.info(f"[MoodleQueue] Reset {count} failed tasks back to PENDING.")
return count
@staticmethod
def get_queue_summary() -> dict:
"""
Returns stats about tasks currently in the queue.
"""
counts = {
'PENDING': 0,
'PROCESSING': 0,
'RETRYING': 0,
'COMPLETED': 0,
'FAILED': 0
}
from sqlalchemy import func
rows = db.session.query(MoodleSyncTask.status, func.count(MoodleSyncTask.id)).group_by(MoodleSyncTask.status).all()
for status, count in rows:
if status in counts:
counts[status] = count
total = sum(counts.values())
return {
'summary': counts,
'total': total,
'pending_total': counts['PENDING'] + counts['RETRYING'] + counts['PROCESSING'],
'failed_dlq': counts['FAILED']
}
moodle_queue_service = MoodleQueueService()