feat: Initial commit for OnEver Drive centralized backup system with Windows PyQt6 agent and Proxmox LXC deployment
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Dict, Any
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.models.models import EventLog
|
||||
from app.ws.manager import ws_manager
|
||||
|
||||
async def log_event(
|
||||
db: AsyncSession,
|
||||
event_type: str,
|
||||
message: str,
|
||||
severity: str = "INFO",
|
||||
client_id: Optional[int] = None,
|
||||
job_id: Optional[int] = None,
|
||||
user_email: Optional[str] = None,
|
||||
ip_address: Optional[str] = None,
|
||||
details: Optional[Dict[str, Any]] = None
|
||||
) -> EventLog:
|
||||
"""Records an audit event in the database and broadcasts it over WebSockets."""
|
||||
details_str = json.dumps(details, default=str) if details else None
|
||||
|
||||
event = EventLog(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
event_type=event_type,
|
||||
severity=severity,
|
||||
client_id=client_id,
|
||||
job_id=job_id,
|
||||
user_email=user_email,
|
||||
ip_address=ip_address,
|
||||
message=message,
|
||||
details_json=details_str
|
||||
)
|
||||
db.add(event)
|
||||
await db.commit()
|
||||
await db.refresh(event)
|
||||
|
||||
# Broadcast real-time event to all Web UI connections
|
||||
await ws_manager.broadcast("EVENT_LOG", {
|
||||
"id": event.id,
|
||||
"timestamp": event.timestamp.isoformat(),
|
||||
"event_type": event.event_type,
|
||||
"severity": event.severity,
|
||||
"client_id": event.client_id,
|
||||
"job_id": event.job_id,
|
||||
"message": event.message
|
||||
})
|
||||
|
||||
return event
|
||||
@@ -0,0 +1,110 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Set
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from app.models.models import BackupJob, BackupFile, Client
|
||||
from app.storage.local import storage_provider
|
||||
from app.services.event_service import log_event
|
||||
|
||||
async def apply_retention_policy(db: AsyncSession, job_id: int) -> int:
|
||||
"""
|
||||
Applies retention rules (daily, weekly, monthly) for a specific backup job.
|
||||
Safely deletes obsolete backup files from storage and database.
|
||||
Returns the number of pruned backup files.
|
||||
"""
|
||||
# Fetch job details
|
||||
result = await db.execute(select(BackupJob).where(BackupJob.id == job_id))
|
||||
job = result.scalar_one_or_none()
|
||||
if not job:
|
||||
return 0
|
||||
|
||||
# Fetch all active backup files for this job ordered by creation date descending
|
||||
result = await db.execute(
|
||||
select(BackupFile)
|
||||
.where(BackupFile.job_id == job_id, BackupFile.is_active == True)
|
||||
.order_by(BackupFile.created_at.desc())
|
||||
)
|
||||
backup_files: List[BackupFile] = result.scalars().all()
|
||||
|
||||
if not backup_files:
|
||||
return 0
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
protected_file_ids: Set[int] = set()
|
||||
|
||||
# Always keep the most recent backup regardless of age to avoid empty repo
|
||||
protected_file_ids.add(backup_files[0].id)
|
||||
|
||||
# 1. Daily retention: Keep 1 backup per calendar day for the last `job.keep_daily` days
|
||||
seen_days = set()
|
||||
for bf in backup_files:
|
||||
day_key = bf.created_at.strftime("%Y-%m-%d")
|
||||
age_days = (now - bf.created_at).total_seconds() / 86400.0
|
||||
if age_days <= job.keep_daily:
|
||||
if day_key not in seen_days:
|
||||
seen_days.add(day_key)
|
||||
protected_file_ids.add(bf.id)
|
||||
|
||||
# 2. Weekly retention: Keep 1 backup per calendar week for the last `job.keep_weekly` weeks
|
||||
seen_weeks = set()
|
||||
for bf in backup_files:
|
||||
week_key = f"{bf.created_at.year}-W{bf.created_at.isocalendar()[1]:02d}"
|
||||
age_weeks = (now - bf.created_at).total_seconds() / (86400.0 * 7)
|
||||
if age_weeks <= job.keep_weekly:
|
||||
if week_key not in seen_weeks:
|
||||
seen_weeks.add(week_key)
|
||||
protected_file_ids.add(bf.id)
|
||||
|
||||
# 3. Monthly retention: Keep 1 backup per calendar month for the last `job.keep_monthly` months
|
||||
seen_months = set()
|
||||
for bf in backup_files:
|
||||
month_key = bf.created_at.strftime("%Y-%m")
|
||||
# Approximate age in months (30 days per month)
|
||||
age_months = (now - bf.created_at).total_seconds() / (86400.0 * 30.4375)
|
||||
if age_months <= job.keep_monthly:
|
||||
if month_key not in seen_months:
|
||||
seen_months.add(month_key)
|
||||
protected_file_ids.add(bf.id)
|
||||
|
||||
# Identify files to delete
|
||||
files_to_delete = [bf for bf in backup_files if bf.id not in protected_file_ids]
|
||||
pruned_count = 0
|
||||
reclaimed_bytes = 0
|
||||
|
||||
for bf in files_to_delete:
|
||||
try:
|
||||
# Delete from physical storage
|
||||
await storage_provider.delete_backup_file(bf.relative_path)
|
||||
bf.is_active = False
|
||||
pruned_count += 1
|
||||
reclaimed_bytes += bf.file_size
|
||||
except Exception as ex:
|
||||
# Log failure but continue processing other files
|
||||
await log_event(
|
||||
db=db,
|
||||
event_type="RETENTION_ERROR",
|
||||
message=f"Failed to delete pruned backup file {bf.filename}: {str(ex)}",
|
||||
severity="WARNING",
|
||||
client_id=job.client_id,
|
||||
job_id=job.id
|
||||
)
|
||||
|
||||
if pruned_count > 0:
|
||||
# Update client storage used counter
|
||||
client_res = await db.execute(select(Client).where(Client.id == job.client_id))
|
||||
client = client_res.scalar_one_or_none()
|
||||
if client:
|
||||
client.storage_used_bytes = max(0, client.storage_used_bytes - reclaimed_bytes)
|
||||
|
||||
await db.commit()
|
||||
|
||||
await log_event(
|
||||
db=db,
|
||||
event_type="RETENTION_APPLIED",
|
||||
message=f"Retention policy applied for job '{job.name}': {pruned_count} obsolete backups removed, {reclaimed_bytes / (1024*1024):.2f} MB reclaimed.",
|
||||
severity="INFO",
|
||||
client_id=job.client_id,
|
||||
job_id=job.id
|
||||
)
|
||||
|
||||
return pruned_count
|
||||
@@ -0,0 +1,277 @@
|
||||
import math
|
||||
from datetime import datetime, timezone
|
||||
from typing import Tuple, List, Dict, Any, Optional
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from app.models.models import Client, BackupJob, BackupSession, BackupChunk, BackupFile
|
||||
from app.storage.local import storage_provider
|
||||
from app.services.event_service import log_event
|
||||
from app.services.retention_service import apply_retention_policy
|
||||
from app.ws.manager import ws_manager
|
||||
|
||||
async def create_or_resume_session(
|
||||
db: AsyncSession,
|
||||
client: Client,
|
||||
filename: str,
|
||||
file_size: int,
|
||||
sha256_full: str,
|
||||
chunk_size: int = 4 * 1024 * 1024,
|
||||
job_id: Optional[int] = None
|
||||
) -> Tuple[BackupSession, List[int]]:
|
||||
"""
|
||||
Initializes a new upload session or resumes an existing incomplete session
|
||||
for the specified file and hash.
|
||||
"""
|
||||
total_chunks = max(1, math.ceil(file_size / chunk_size))
|
||||
|
||||
# Check for existing incomplete session for this client and file hash
|
||||
query = (
|
||||
select(BackupSession)
|
||||
.where(
|
||||
BackupSession.client_id == client.id,
|
||||
BackupSession.sha256_full == sha256_full,
|
||||
BackupSession.file_size == file_size,
|
||||
BackupSession.status.in_(["PENDING", "UPLOADING"])
|
||||
)
|
||||
)
|
||||
result = await db.execute(query)
|
||||
session = result.scalar_one_or_none()
|
||||
|
||||
if session:
|
||||
# Resume existing session
|
||||
received_chunks = await storage_provider.get_received_chunks(session.session_code)
|
||||
session.received_chunks_count = len(received_chunks)
|
||||
await db.commit()
|
||||
await db.refresh(session)
|
||||
return session, received_chunks
|
||||
|
||||
# Create new upload session
|
||||
session = BackupSession(
|
||||
client_id=client.id,
|
||||
job_id=job_id,
|
||||
filename=filename,
|
||||
file_size=file_size,
|
||||
chunk_size=chunk_size,
|
||||
total_chunks=total_chunks,
|
||||
received_chunks_count=0,
|
||||
sha256_full=sha256_full,
|
||||
status="UPLOADING",
|
||||
started_at=datetime.now(timezone.utc)
|
||||
)
|
||||
db.add(session)
|
||||
await db.commit()
|
||||
await db.refresh(session)
|
||||
|
||||
# Initialize temporary storage
|
||||
await storage_provider.init_session_storage(session.session_code)
|
||||
|
||||
await log_event(
|
||||
db=db,
|
||||
event_type="BACKUP_STARTED",
|
||||
message=f"Upload session initiated for '{filename}' ({file_size / (1024*1024):.2f} MB, {total_chunks} chunks).",
|
||||
severity="INFO",
|
||||
client_id=client.id,
|
||||
job_id=job_id,
|
||||
details={"session_code": session.session_code, "total_chunks": total_chunks}
|
||||
)
|
||||
|
||||
return session, []
|
||||
|
||||
async def process_chunk_upload(
|
||||
db: AsyncSession,
|
||||
session: BackupSession,
|
||||
chunk_index: int,
|
||||
chunk_data: bytes,
|
||||
chunk_sha256: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Saves a chunk to temporary storage, registers chunk in database,
|
||||
and broadcasts live progress telemetry.
|
||||
"""
|
||||
if session.status not in ["PENDING", "UPLOADING"]:
|
||||
raise ValueError(f"Cannot upload chunk: session is currently in state {session.status}")
|
||||
|
||||
if chunk_index < 0 or chunk_index >= session.total_chunks:
|
||||
raise ValueError(f"Invalid chunk_index {chunk_index}. Session total chunks: {session.total_chunks}")
|
||||
|
||||
# Save to storage (performs chunk SHA-256 verification if provided)
|
||||
await storage_provider.save_chunk(
|
||||
session_code=session.session_code,
|
||||
chunk_index=chunk_index,
|
||||
chunk_data=chunk_data,
|
||||
expected_sha256=chunk_sha256
|
||||
)
|
||||
|
||||
# Record in database
|
||||
result = await db.execute(
|
||||
select(BackupChunk).where(
|
||||
BackupChunk.session_id == session.id,
|
||||
BackupChunk.chunk_index == chunk_index
|
||||
)
|
||||
)
|
||||
chunk_rec = result.scalar_one_or_none()
|
||||
|
||||
if not chunk_rec:
|
||||
chunk_rec = BackupChunk(
|
||||
session_id=session.id,
|
||||
chunk_index=chunk_index,
|
||||
chunk_size=len(chunk_data),
|
||||
sha256=chunk_sha256 or "",
|
||||
is_received=True,
|
||||
received_at=datetime.now(timezone.utc)
|
||||
)
|
||||
db.add(chunk_rec)
|
||||
else:
|
||||
chunk_rec.is_received = True
|
||||
chunk_rec.received_at = datetime.now(timezone.utc)
|
||||
|
||||
# Count received chunks
|
||||
received_list = await storage_provider.get_received_chunks(session.session_code)
|
||||
session.received_chunks_count = len(received_list)
|
||||
await db.commit()
|
||||
|
||||
progress_pct = round((session.received_chunks_count / session.total_chunks) * 100, 2)
|
||||
|
||||
# Broadcast live telemetry over WebSocket
|
||||
await ws_manager.broadcast("UPLOAD_PROGRESS", {
|
||||
"session_code": session.session_code,
|
||||
"filename": session.filename,
|
||||
"client_id": session.client_id,
|
||||
"chunk_index": chunk_index,
|
||||
"received_chunks": session.received_chunks_count,
|
||||
"total_chunks": session.total_chunks,
|
||||
"progress_percent": progress_pct
|
||||
})
|
||||
|
||||
return {
|
||||
"chunk_index": chunk_index,
|
||||
"is_received": True,
|
||||
"total_received": session.received_chunks_count,
|
||||
"total_chunks": session.total_chunks,
|
||||
"progress_percent": progress_pct
|
||||
}
|
||||
|
||||
async def get_session_status_info(
|
||||
db: AsyncSession,
|
||||
session: BackupSession
|
||||
) -> Dict[str, Any]:
|
||||
"""Returns detailed session status and lists of received / missing chunks."""
|
||||
received = await storage_provider.get_received_chunks(session.session_code)
|
||||
received_set = set(received)
|
||||
missing = [i for i in range(session.total_chunks) if i not in received_set]
|
||||
progress_pct = round((len(received) / session.total_chunks) * 100, 2)
|
||||
|
||||
return {
|
||||
"session_code": session.session_code,
|
||||
"filename": session.filename,
|
||||
"file_size": session.file_size,
|
||||
"chunk_size": session.chunk_size,
|
||||
"total_chunks": session.total_chunks,
|
||||
"received_chunks": received,
|
||||
"missing_chunks": missing,
|
||||
"status": session.status,
|
||||
"progress_percent": progress_pct
|
||||
}
|
||||
|
||||
async def complete_session(
|
||||
db: AsyncSession,
|
||||
session: BackupSession
|
||||
) -> BackupFile:
|
||||
"""
|
||||
Assembles chunks into final storage, verifies SHA-256 integrity,
|
||||
updates client stats, applies retention policy, and logs completion.
|
||||
"""
|
||||
# Fetch client and job codes for directory naming
|
||||
client_res = await db.execute(select(Client).where(Client.id == session.client_id))
|
||||
client = client_res.scalar_one_or_none()
|
||||
if not client:
|
||||
raise ValueError(f"Client {session.client_id} not found")
|
||||
|
||||
job_code = "DEFAULT"
|
||||
if session.job_id:
|
||||
job_res = await db.execute(select(BackupJob).where(BackupJob.id == session.job_id))
|
||||
job = job_res.scalar_one_or_none()
|
||||
if job:
|
||||
job_code = job.job_code
|
||||
|
||||
session.status = "ASSEMBLING"
|
||||
await db.commit()
|
||||
|
||||
try:
|
||||
# Assemble and verify streaming SHA-256
|
||||
rel_path, final_sha256, total_bytes = await storage_provider.assemble_file(
|
||||
session_code=session.session_code,
|
||||
client_code=client.client_code,
|
||||
job_code=job_code,
|
||||
filename=session.filename,
|
||||
total_chunks=session.total_chunks,
|
||||
expected_sha256=session.sha256_full
|
||||
)
|
||||
|
||||
session.status = "SUCCESS"
|
||||
session.completed_at = datetime.now(timezone.utc)
|
||||
|
||||
# Create BackupFile record
|
||||
backup_file = BackupFile(
|
||||
client_id=client.id,
|
||||
job_id=session.job_id,
|
||||
session_id=session.id,
|
||||
filename=session.filename,
|
||||
relative_path=rel_path,
|
||||
file_size=total_bytes,
|
||||
sha256=final_sha256,
|
||||
retention_tag="DAILY",
|
||||
is_active=True,
|
||||
created_at=datetime.now(timezone.utc)
|
||||
)
|
||||
db.add(backup_file)
|
||||
|
||||
# Update client storage and last backup timestamp
|
||||
client.storage_used_bytes += total_bytes
|
||||
client.last_backup_at = datetime.now(timezone.utc)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(backup_file)
|
||||
|
||||
# Log completion event
|
||||
await log_event(
|
||||
db=db,
|
||||
event_type="BACKUP_COMPLETED",
|
||||
message=f"Backup successfully verified & stored: '{session.filename}' ({total_bytes / (1024*1024):.2f} MB). SHA-256: {final_sha256[:16]}...",
|
||||
severity="INFO",
|
||||
client_id=client.id,
|
||||
job_id=session.job_id,
|
||||
details={"sha256": final_sha256, "file_size": total_bytes, "path": rel_path}
|
||||
)
|
||||
|
||||
# Apply retention policy if associated with a job
|
||||
if session.job_id:
|
||||
await apply_retention_policy(db, session.job_id)
|
||||
|
||||
# Broadcast completion
|
||||
await ws_manager.broadcast("UPLOAD_COMPLETED", {
|
||||
"session_code": session.session_code,
|
||||
"filename": session.filename,
|
||||
"client_id": client.id,
|
||||
"file_size": total_bytes,
|
||||
"sha256": final_sha256,
|
||||
"status": "SUCCESS"
|
||||
})
|
||||
|
||||
return backup_file
|
||||
|
||||
except Exception as ex:
|
||||
session.status = "FAILED"
|
||||
session.error_message = str(ex)
|
||||
await db.commit()
|
||||
|
||||
await log_event(
|
||||
db=db,
|
||||
event_type="BACKUP_FAILED",
|
||||
message=f"Backup assembly/verification failed for '{session.filename}': {str(ex)}",
|
||||
severity="ERROR",
|
||||
client_id=client.id,
|
||||
job_id=session.job_id,
|
||||
details={"error": str(ex)}
|
||||
)
|
||||
raise ex
|
||||
Reference in New Issue
Block a user