feat: Initial commit for OnEver Drive centralized backup system with Windows PyQt6 agent and Proxmox LXC deployment

This commit is contained in:
2026-08-13 19:33:44 -03:00
commit 1bfb808c79
77 changed files with 10675 additions and 0 deletions
+72
View File
@@ -0,0 +1,72 @@
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.core.database import get_db
from app.core.security import verify_password, get_password_hash, create_access_token
from app.models.models import User
from app.schemas.schemas import LoginRequest, TokenResponse, UserResponse
from app.api.deps import get_current_user
from app.services.event_service import log_event
router = APIRouter(prefix="/auth", tags=["Authentication"])
@router.post("/login", response_model=TokenResponse)
async def login(credentials: LoginRequest, db: AsyncSession = Depends(get_db)):
result = await db.execute(select(User).where(User.email == credentials.email.strip().lower()))
user = result.scalar_one_or_none()
if not user or not verify_password(credentials.password, user.hashed_password):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect email or password"
)
if not user.is_active:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="User account is deactivated"
)
access_token = create_access_token(subject=user.id, role=user.role)
await log_event(
db=db,
event_type="LOGIN",
message=f"User {user.email} logged in successfully.",
severity="INFO",
user_email=user.email
)
return {
"access_token": access_token,
"token_type": "bearer",
"user": {
"id": user.id,
"email": user.email,
"full_name": user.full_name,
"role": user.role
}
}
@router.get("/me", response_model=UserResponse)
async def get_me(current_user: User = Depends(get_current_user)):
return current_user
@router.post("/seed-admin")
async def seed_admin(db: AsyncSession = Depends(get_db)):
"""Creates default admin if no users exist in the database."""
result = await db.execute(select(User))
first_user = result.scalar_one_or_none()
if first_user:
return {"message": "Admin already exists"}
admin = User(
email="admin@oneverdrive.local",
hashed_password=get_password_hash("Admin1234!"),
full_name="System Administrator",
role="ADMIN",
is_active=True
)
db.add(admin)
await db.commit()
return {"message": "Default admin created: admin@oneverdrive.local / Admin1234!"}
+116
View File
@@ -0,0 +1,116 @@
import os
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.responses import FileResponse
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, desc
from app.core.database import get_db
from app.models.models import User, BackupFile, Client
from app.schemas.schemas import BackupFileResponse
from app.api.deps import get_current_user, require_admin
from app.storage.local import storage_provider
from app.services.event_service import log_event
router = APIRouter(prefix="/backups", tags=["Backups & Restore"])
@router.get("", response_model=List[BackupFileResponse])
async def list_backups(
client_id: Optional[int] = None,
job_id: Optional[int] = None,
limit: int = 100,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
query = select(BackupFile).where(BackupFile.is_active == True)
if client_id:
query = query.where(BackupFile.client_id == client_id)
if job_id:
query = query.where(BackupFile.job_id == job_id)
query = query.order_by(desc(BackupFile.created_at)).limit(limit)
result = await db.execute(query)
return result.scalars().all()
@router.get("/{backup_id}", response_model=BackupFileResponse)
async def get_backup(
backup_id: int,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
res = await db.execute(select(BackupFile).where(BackupFile.id == backup_id))
bf = res.scalar_one_or_none()
if not bf:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Backup file not found")
return bf
@router.get("/{backup_id}/download")
async def download_backup(
backup_id: int,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""Allows downloading a backup file directly from the Web interface."""
res = await db.execute(select(BackupFile).where(BackupFile.id == backup_id))
bf = res.scalar_one_or_none()
if not bf or not bf.is_active:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Backup file not found")
try:
abs_path = await storage_provider.get_file_path(bf.relative_path)
except Exception as ex:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(ex))
await log_event(
db=db,
event_type="RESTORE_STARTED",
message=f"Backup download initiated for '{bf.filename}' ({bf.file_size / (1024*1024):.2f} MB).",
severity="INFO",
client_id=bf.client_id,
job_id=bf.job_id,
user_email=current_user.email
)
return FileResponse(
path=abs_path,
filename=bf.filename,
media_type="application/octet-stream"
)
@router.delete("/{backup_id}")
async def delete_backup(
backup_id: int,
db: AsyncSession = Depends(get_db),
admin_user: User = Depends(require_admin)
):
res = await db.execute(select(BackupFile).where(BackupFile.id == backup_id))
bf = res.scalar_one_or_none()
if not bf:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Backup file not found")
try:
await storage_provider.delete_backup_file(bf.relative_path)
except Exception:
pass
bf.is_active = False
# Update client storage counter
client_res = await db.execute(select(Client).where(Client.id == bf.client_id))
client = client_res.scalar_one_or_none()
if client:
client.storage_used_bytes = max(0, client.storage_used_bytes - bf.file_size)
await db.commit()
await log_event(
db=db,
event_type="FILE_DELETED",
message=f"Manual deletion of backup '{bf.filename}'.",
severity="WARNING",
client_id=bf.client_id,
job_id=bf.job_id,
user_email=admin_user.email
)
return {"message": "Backup file deleted"}
+245
View File
@@ -0,0 +1,245 @@
import uuid
from datetime import datetime, timedelta, timezone
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, status, Request
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func, desc
from app.core.database import get_db
from app.core.security import generate_registration_code, generate_device_token, hash_token
from app.models.models import User, Client, ClientCredential, RegistrationCode
from app.schemas.schemas import (
ClientResponse, ClientRegisterRequest, ClientRegisterResponse,
RegistrationCodeCreate, RegistrationCodeResponse, ClientHeartbeatRequest
)
from app.api.deps import get_current_user, require_admin, get_current_client
from app.services.event_service import log_event
from app.ws.manager import ws_manager
router = APIRouter(prefix="/clients", tags=["Clients"])
@router.get("", response_model=List[ClientResponse])
async def list_clients(
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
result = await db.execute(select(Client).order_by(desc(Client.created_at)))
return result.scalars().all()
@router.post("/registration-code", response_model=RegistrationCodeResponse)
async def create_registration_code(
payload: RegistrationCodeCreate,
db: AsyncSession = Depends(get_db),
admin_user: User = Depends(require_admin)
):
code_str = generate_registration_code()
expires = datetime.now(timezone.utc) + timedelta(hours=payload.expires_in_hours)
reg_code = RegistrationCode(
code=code_str,
client_name_hint=payload.client_name_hint,
expires_at=expires,
is_used=False
)
db.add(reg_code)
await db.commit()
await db.refresh(reg_code)
await log_event(
db=db,
event_type="REGISTRATION_CODE_GENERATED",
message=f"Generated registration code {code_str} (hint: {payload.client_name_hint or 'None'}).",
severity="INFO",
user_email=admin_user.email
)
return reg_code
@router.post("/register", response_model=ClientRegisterResponse)
async def register_client(
payload: ClientRegisterRequest,
request: Request,
db: AsyncSession = Depends(get_db)
):
"""Called by the Windows Agent during initial setup to register against the server."""
# Find valid registration code
now = datetime.now(timezone.utc)
res = await db.execute(
select(RegistrationCode).where(
RegistrationCode.code == payload.registration_code.strip().upper(),
RegistrationCode.is_used == False,
RegistrationCode.expires_at > now
)
)
reg_code = res.scalar_one_or_none()
if not reg_code:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Invalid, expired, or already used registration code."
)
# Determine next client code (e.g., CLIENT-0001)
count_res = await db.execute(select(func.count(Client.id)))
client_count = count_res.scalar() or 0
client_code = f"CLIENT-{client_count + 1:04d}"
client_ip = request.client.host if request.client else None
# Create Client
client = Client(
client_code=client_code,
name=payload.name or reg_code.client_name_hint or payload.hostname,
hostname=payload.hostname,
os_info=payload.os_info,
ip_address=client_ip,
agent_version=payload.agent_version,
status="ONLINE",
last_seen_at=now,
is_active=True
)
db.add(client)
await db.flush()
# Generate device unique ID and secret token
device_id = str(uuid.uuid4())
device_token = generate_device_token()
token_hash = hash_token(device_token)
credential = ClientCredential(
client_id=client.id,
device_id=device_id,
token_hash=token_hash,
name=f"{payload.hostname} Agent",
is_revoked=False,
created_at=now,
last_used_at=now
)
db.add(credential)
# Mark registration code as used
reg_code.is_used = True
await db.commit()
await log_event(
db=db,
event_type="CLIENT_REGISTERED",
message=f"Windows client registered: {client.name} ({client.client_code}, Hostname: {client.hostname}, IP: {client_ip})",
severity="INFO",
client_id=client.id,
ip_address=client_ip,
details={"device_id": device_id, "os_info": payload.os_info}
)
await ws_manager.broadcast("CLIENT_REGISTERED", {
"id": client.id,
"client_code": client.client_code,
"name": client.name,
"hostname": client.hostname,
"status": client.status
})
return {
"client_code": client.client_code,
"device_id": device_id,
"device_token": device_token,
"name": client.name,
"server_time": now
}
@router.post("/{client_id}/heartbeat")
async def client_heartbeat(
client_id: int,
payload: ClientHeartbeatRequest,
request: Request,
db: AsyncSession = Depends(get_db),
current_client: Client = Depends(get_current_client)
):
"""Heartbeat endpoint invoked periodically by the Windows Agent."""
if current_client.id != client_id:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Client ID mismatch")
now = datetime.now(timezone.utc)
current_client.last_seen_at = now
current_client.status = payload.status
if payload.agent_version:
current_client.agent_version = payload.agent_version
client_ip = payload.ip_address or (request.client.host if request.client else None)
if client_ip:
current_client.ip_address = client_ip
await db.commit()
await ws_manager.broadcast("CLIENT_HEARTBEAT", {
"client_id": current_client.id,
"status": current_client.status,
"last_seen_at": now.isoformat()
})
return {"status": "ok", "server_time": now}
@router.get("/{client_id}", response_model=ClientResponse)
async def get_client(
client_id: int,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
res = await db.execute(select(Client).where(Client.id == client_id))
client = res.scalar_one_or_none()
if not client:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Client not found")
return client
@router.post("/{client_id}/revoke")
async def revoke_client_credentials(
client_id: int,
db: AsyncSession = Depends(get_db),
admin_user: User = Depends(require_admin)
):
"""Revokes all active authentication tokens for a client."""
res = await db.execute(select(ClientCredential).where(ClientCredential.client_id == client_id))
creds = res.scalars().all()
for cred in creds:
cred.is_revoked = True
client_res = await db.execute(select(Client).where(Client.id == client_id))
client = client_res.scalar_one_or_none()
if client:
client.status = "OFFLINE"
await db.commit()
await log_event(
db=db,
event_type="CREDENTIALS_REVOKED",
message=f"Revoked credentials for client ID {client_id}.",
severity="WARNING",
client_id=client_id,
user_email=admin_user.email
)
return {"message": "Client credentials successfully revoked"}
@router.delete("/{client_id}")
async def delete_client(
client_id: int,
db: AsyncSession = Depends(get_db),
admin_user: User = Depends(require_admin)
):
res = await db.execute(select(Client).where(Client.id == client_id))
client = res.scalar_one_or_none()
if not client:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Client not found")
await db.delete(client)
await db.commit()
await log_event(
db=db,
event_type="CLIENT_DELETED",
message=f"Deleted client {client.name} ({client.client_code}).",
severity="WARNING",
user_email=admin_user.email
)
return {"message": f"Client {client.client_code} deleted"}
+87
View File
@@ -0,0 +1,87 @@
from typing import Optional
from fastapi import Depends, HTTPException, status, Header
from fastapi.security import OAuth2PasswordBearer
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.core.database import get_db
from app.core.security import decode_access_token, hash_token
from app.models.models import User, Client, ClientCredential
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_error=False)
async def get_current_user(
token: Optional[str] = Depends(oauth2_scheme),
db: AsyncSession = Depends(get_db)
) -> User:
"""Authenticates web UI users via JWT Bearer token."""
if not token:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Authentication required",
headers={"WWW-Authenticate": "Bearer"}
)
payload = decode_access_token(token)
if not payload or "sub" not in payload:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid or expired authentication token",
headers={"WWW-Authenticate": "Bearer"}
)
user_id = int(payload["sub"])
result = await db.execute(select(User).where(User.id == user_id, User.is_active == True))
user = result.scalar_one_or_none()
if not user:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="User not found or deactivated"
)
return user
async def require_admin(user: User = Depends(get_current_user)) -> User:
"""Ensures current user has ADMIN role."""
if user.role != "ADMIN":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Admin privilege required"
)
return user
async def get_current_client(
x_device_id: Optional[str] = Header(None, alias="X-Device-Id"),
x_device_token: Optional[str] = Header(None, alias="X-Device-Token"),
db: AsyncSession = Depends(get_db)
) -> Client:
"""Authenticates Windows Agent devices via individual device ID and secret token."""
if not x_device_id or not x_device_token:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Device authentication headers (X-Device-Id, X-Device-Token) required"
)
token_hash = hash_token(x_device_token)
result = await db.execute(
select(ClientCredential)
.where(
ClientCredential.device_id == x_device_id,
ClientCredential.token_hash == token_hash,
ClientCredential.is_revoked == False
)
)
credential = result.scalar_one_or_none()
if not credential:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid or revoked device credentials"
)
client_res = await db.execute(select(Client).where(Client.id == credential.client_id, Client.is_active == True))
client = client_res.scalar_one_or_none()
if not client:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Client device not found or inactive"
)
return client
+32
View File
@@ -0,0 +1,32 @@
from typing import List, Optional
from fastapi import APIRouter, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, desc
from app.core.database import get_db
from app.models.models import EventLog, User
from app.schemas.schemas import EventLogResponse
from app.api.deps import get_current_user
router = APIRouter(prefix="/events", tags=["Audit & Events"])
@router.get("", response_model=List[EventLogResponse])
async def list_events(
client_id: Optional[int] = None,
job_id: Optional[int] = None,
event_type: Optional[str] = None,
limit: int = Query(50, le=200),
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
query = select(EventLog)
if client_id:
query = query.where(EventLog.client_id == client_id)
if job_id:
query = query.where(EventLog.job_id == job_id)
if event_type:
query = query.where(EventLog.event_type == event_type)
query = query.order_by(desc(EventLog.timestamp)).limit(limit)
result = await db.execute(query)
return result.scalars().all()
+175
View File
@@ -0,0 +1,175 @@
from typing import List, Optional
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func, desc
from app.core.database import get_db
from app.models.models import User, Client, BackupJob
from app.schemas.schemas import JobCreate, JobUpdate, JobResponse
from app.api.deps import get_current_user, require_admin, get_current_client
from app.services.event_service import log_event
from app.ws.manager import ws_manager
router = APIRouter(prefix="/jobs", tags=["Backup Jobs"])
@router.get("", response_model=List[JobResponse])
async def list_jobs(
client_id: Optional[int] = None,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
query = select(BackupJob)
if client_id:
query = query.where(BackupJob.client_id == client_id)
query = query.order_by(desc(BackupJob.created_at))
result = await db.execute(query)
return result.scalars().all()
@router.get("/agent/assigned", response_model=List[JobResponse])
async def get_agent_jobs(
db: AsyncSession = Depends(get_db),
current_client: Client = Depends(get_current_client)
):
"""Called by the Windows Agent to query its active backup jobs."""
query = select(BackupJob).where(BackupJob.client_id == current_client.id, BackupJob.is_active == True)
result = await db.execute(query)
return result.scalars().all()
@router.post("", response_model=JobResponse)
async def create_job(
payload: JobCreate,
db: AsyncSession = Depends(get_db),
admin_user: User = Depends(require_admin)
):
# Verify client exists
client_res = await db.execute(select(Client).where(Client.id == payload.client_id))
client = client_res.scalar_one_or_none()
if not client:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Client not found")
count_res = await db.execute(select(func.count(BackupJob.id)))
job_count = count_res.scalar() or 0
job_code = f"JOB-{job_count + 1:03d}"
job = BackupJob(
job_code=job_code,
client_id=payload.client_id,
name=payload.name,
source_path=payload.source_path,
file_patterns=payload.file_patterns,
schedule_cron=payload.schedule_cron,
keep_daily=payload.keep_daily,
keep_weekly=payload.keep_weekly,
keep_monthly=payload.keep_monthly,
min_stable_time_seconds=payload.min_stable_time_seconds,
status="IDLE",
is_active=True
)
db.add(job)
await db.commit()
await db.refresh(job)
await log_event(
db=db,
event_type="JOB_CREATED",
message=f"Created backup job '{job.name}' ({job.job_code}) for client {client.name}.",
severity="INFO",
client_id=client.id,
job_id=job.id,
user_email=admin_user.email
)
return job
@router.get("/{job_id}", response_model=JobResponse)
async def get_job(
job_id: int,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
res = await db.execute(select(BackupJob).where(BackupJob.id == job_id))
job = res.scalar_one_or_none()
if not job:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found")
return job
@router.put("/{job_id}", response_model=JobResponse)
async def update_job(
job_id: int,
payload: JobUpdate,
db: AsyncSession = Depends(get_db),
admin_user: User = Depends(require_admin)
):
res = await db.execute(select(BackupJob).where(BackupJob.id == job_id))
job = res.scalar_one_or_none()
if not job:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found")
for field, value in payload.model_dump(exclude_unset=True).items():
setattr(job, field, value)
await db.commit()
await db.refresh(job)
await log_event(
db=db,
event_type="CONFIG_CHANGED",
message=f"Updated backup job '{job.name}' ({job.job_code}).",
severity="INFO",
client_id=job.client_id,
job_id=job.id,
user_email=admin_user.email
)
return job
@router.delete("/{job_id}")
async def delete_job(
job_id: int,
db: AsyncSession = Depends(get_db),
admin_user: User = Depends(require_admin)
):
res = await db.execute(select(BackupJob).where(BackupJob.id == job_id))
job = res.scalar_one_or_none()
if not job:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found")
await db.delete(job)
await db.commit()
await log_event(
db=db,
event_type="JOB_DELETED",
message=f"Deleted backup job {job.job_code}.",
severity="WARNING",
client_id=job.client_id,
user_email=admin_user.email
)
return {"message": f"Job {job.job_code} deleted"}
@router.post("/{job_id}/trigger")
async def trigger_job(
job_id: int,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
"""Notifies the connected agent to start executing this backup job immediately."""
res = await db.execute(select(BackupJob).where(BackupJob.id == job_id))
job = res.scalar_one_or_none()
if not job:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found")
job.status = "QUEUED"
await db.commit()
# Broadcast event so the agent / web UI knows the job was triggered
await ws_manager.broadcast("JOB_TRIGGERED", {
"job_id": job.id,
"job_code": job.job_code,
"client_id": job.client_id,
"timestamp": datetime.now(timezone.utc).isoformat()
})
return {"message": f"Job {job.job_code} triggered"}
+81
View File
@@ -0,0 +1,81 @@
from datetime import datetime, timezone, timedelta
from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func
from app.core.database import get_db
from app.models.models import Client, BackupJob, BackupSession, User
from app.schemas.schemas import DashboardStatsResponse
from app.api.deps import get_current_user
from app.storage.local import storage_provider
router = APIRouter(prefix="/stats", tags=["Dashboard Statistics"])
@router.get("", response_model=DashboardStatsResponse)
async def get_dashboard_stats(
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user)
):
# Clients aggregation
total_clients_res = await db.execute(select(func.count(Client.id)).where(Client.is_active == True))
total_clients = total_clients_res.scalar() or 0
# Consider online if last_seen_at was in the last 5 minutes
cutoff_online = datetime.now(timezone.utc) - timedelta(minutes=5)
online_clients_res = await db.execute(
select(func.count(Client.id)).where(
Client.is_active == True,
Client.status == "ONLINE",
Client.last_seen_at >= cutoff_online
)
)
online_clients = online_clients_res.scalar() or 0
offline_clients = max(0, total_clients - online_clients)
# Jobs count
total_jobs_res = await db.execute(select(func.count(BackupJob.id)).where(BackupJob.is_active == True))
total_jobs = total_jobs_res.scalar() or 0
# Backups today
today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
backups_today_res = await db.execute(
select(func.count(BackupSession.id)).where(BackupSession.started_at >= today_start)
)
backups_today = backups_today_res.scalar() or 0
backups_success_res = await db.execute(
select(func.count(BackupSession.id)).where(
BackupSession.started_at >= today_start,
BackupSession.status == "SUCCESS"
)
)
backups_success = backups_success_res.scalar() or 0
backups_failed_res = await db.execute(
select(func.count(BackupSession.id)).where(
BackupSession.started_at >= today_start,
BackupSession.status == "FAILED"
)
)
backups_failed = backups_failed_res.scalar() or 0
# Active uploads
active_uploads_res = await db.execute(
select(func.count(BackupSession.id)).where(BackupSession.status.in_(["PENDING", "UPLOADING", "ASSEMBLING"]))
)
active_uploads = active_uploads_res.scalar() or 0
# Storage metrics
storage_stats = await storage_provider.get_storage_stats()
return {
"total_clients": total_clients,
"online_clients": online_clients,
"offline_clients": offline_clients,
"total_jobs": total_jobs,
"backups_today_count": backups_today,
"backups_today_success": backups_success,
"backups_today_failed": backups_failed,
"active_uploads_count": active_uploads,
"storage": storage_stats
}
+153
View File
@@ -0,0 +1,153 @@
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, status, Header, Request, Query, UploadFile, File
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.core.database import get_db
from app.models.models import Client, BackupSession
from app.schemas.schemas import (
UploadSessionInitRequest, UploadSessionInitResponse,
UploadSessionStatusResponse, ChunkUploadResponse,
UploadSessionCompleteResponse
)
from app.api.deps import get_current_client
from app.services.upload_service import (
create_or_resume_session, process_chunk_upload,
get_session_status_info, complete_session
)
router = APIRouter(prefix="/upload", tags=["Chunk Upload Engine"])
@router.post("/session", response_model=UploadSessionInitResponse)
async def init_session(
payload: UploadSessionInitRequest,
db: AsyncSession = Depends(get_db),
current_client: Client = Depends(get_current_client)
):
"""
Initializes a new upload session or resumes an existing incomplete session
for the specified file. Returns list of previously received chunks so the agent
only transmits the remaining blocks.
"""
session, received_chunks = await create_or_resume_session(
db=db,
client=current_client,
filename=payload.filename,
file_size=payload.file_size,
sha256_full=payload.sha256,
chunk_size=payload.chunk_size,
job_id=payload.job_id
)
return {
"session_code": session.session_code,
"filename": session.filename,
"file_size": session.file_size,
"chunk_size": session.chunk_size,
"total_chunks": session.total_chunks,
"received_chunks": received_chunks,
"status": session.status
}
@router.post("/{session_code}/chunk", response_model=ChunkUploadResponse)
async def upload_chunk(
session_code: str,
request: Request,
chunk_index: int = Query(..., description="0-indexed chunk number"),
chunk_sha256: Optional[str] = Query(None, description="SHA-256 hash of this specific chunk"),
x_chunk_index: Optional[int] = Header(None, alias="X-Chunk-Index"),
x_chunk_sha256: Optional[str] = Header(None, alias="X-Chunk-SHA256"),
db: AsyncSession = Depends(get_db),
current_client: Client = Depends(get_current_client)
):
"""
Receives and stores a single chunk of data for an active upload session.
Accepts raw binary body directly via streaming.
"""
idx = x_chunk_index if x_chunk_index is not None else chunk_index
sha = x_chunk_sha256 or chunk_sha256
res = await db.execute(
select(BackupSession).where(
BackupSession.session_code == session_code,
BackupSession.client_id == current_client.id
)
)
session = res.scalar_one_or_none()
if not session:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
chunk_data = await request.body()
if not chunk_data:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Empty chunk payload")
try:
result = await process_chunk_upload(
db=db,
session=session,
chunk_index=idx,
chunk_data=chunk_data,
chunk_sha256=sha
)
return result
except ValueError as ex:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(ex))
@router.get("/{session_code}/status", response_model=UploadSessionStatusResponse)
async def get_session_status(
session_code: str,
db: AsyncSession = Depends(get_db),
current_client: Client = Depends(get_current_client)
):
"""
Returns the current status of an upload session, including the list of received
chunks and missing chunks for easy re-connection and resumption.
"""
res = await db.execute(
select(BackupSession).where(
BackupSession.session_code == session_code,
BackupSession.client_id == current_client.id
)
)
session = res.scalar_one_or_none()
if not session:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
return await get_session_status_info(db, session)
@router.post("/{session_code}/complete", response_model=UploadSessionCompleteResponse)
async def complete_upload(
session_code: str,
db: AsyncSession = Depends(get_db),
current_client: Client = Depends(get_current_client)
):
"""
Triggers sequential file assembly and final SHA-256 integrity verification.
If integrity passes, the backup file is registered and retention policies are applied.
"""
res = await db.execute(
select(BackupSession).where(
BackupSession.session_code == session_code,
BackupSession.client_id == current_client.id
)
)
session = res.scalar_one_or_none()
if not session:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
try:
backup_file = await complete_session(db, session)
return {
"session_code": session.session_code,
"filename": backup_file.filename,
"relative_path": backup_file.relative_path,
"file_size": backup_file.file_size,
"sha256": backup_file.sha256,
"status": "SUCCESS",
"completed_at": backup_file.created_at
}
except Exception as ex:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Integrity check or assembly failed: {str(ex)}"
)
+44
View File
@@ -0,0 +1,44 @@
import os
from pathlib import Path
from pydantic_settings import BaseSettings
BASE_DIR = Path(__file__).resolve().parent.parent.parent
class Settings(BaseSettings):
PROJECT_NAME: str = "OnEver Drive"
VERSION: str = "1.0.0"
API_V1_PREFIX: str = "/api"
# Environment
ENVIRONMENT: str = "development"
DEBUG: bool = True
# Database: Default to SQLite for seamless local dev & test, configurable to PostgreSQL in production
DATABASE_URL: str = f"sqlite+aiosqlite:///{BASE_DIR}/onever_drive.db"
# Storage settings
STORAGE_ROOT: str = str(BASE_DIR / "storage" / "backups")
STORAGE_TEMP_ROOT: str = str(BASE_DIR / "storage" / "temp")
# Security
SECRET_KEY: str = "onever-drive-super-secret-key-change-in-production-2026"
ALGORITHM: str = "HS256"
ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 7 # 7 days
# Upload & Transfer settings
DEFAULT_CHUNK_SIZE: int = 4 * 1024 * 1024 # 4 MB
MAX_CHUNK_SIZE: int = 16 * 1024 * 1024 # 16 MB
MIN_STABLE_TIME_SECONDS: int = 60 # 60s stability window for locked files
# Retention Defaults
DEFAULT_RETENTION_DAILY: int = 7
DEFAULT_RETENTION_WEEKLY: int = 4
DEFAULT_RETENTION_MONTHLY: int = 12
model_config = {"env_file": ".env", "extra": "allow"}
settings = Settings()
# Ensure storage directories exist
os.makedirs(settings.STORAGE_ROOT, exist_ok=True)
os.makedirs(settings.STORAGE_TEMP_ROOT, exist_ok=True)
+34
View File
@@ -0,0 +1,34 @@
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
from sqlalchemy.orm import declarative_base
from app.core.config import settings
# Configure async engine
engine = create_async_engine(
settings.DATABASE_URL,
echo=False,
future=True,
# SQLite-specific optimization if running SQLite
connect_args={"check_same_thread": False} if "sqlite" in settings.DATABASE_URL else {}
)
AsyncSessionLocal = async_sessionmaker(
bind=engine,
class_=AsyncSession,
expire_on_commit=False,
autoflush=False
)
Base = declarative_base()
async def get_db():
"""FastAPI dependency for obtaining async database sessions."""
async with AsyncSessionLocal() as session:
try:
yield session
finally:
await session.close()
async def init_db():
"""Initializes database tables if they do not already exist."""
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
+57
View File
@@ -0,0 +1,57 @@
import secrets
import hashlib
from datetime import datetime, timedelta, timezone
from typing import Optional, Any
import jwt
import bcrypt
from app.core.config import settings
def get_password_hash(password: str) -> str:
"""Hashes a password with bcrypt."""
salt = bcrypt.gensalt()
return bcrypt.hashpw(password.encode("utf-8"), salt).decode("utf-8")
def verify_password(plain_password: str, hashed_password: str) -> bool:
"""Verifies a plain password against a bcrypt hash."""
try:
return bcrypt.checkpw(plain_password.encode("utf-8"), hashed_password.encode("utf-8"))
except Exception:
return False
def create_access_token(subject: Any, role: str = "ADMIN", expires_delta: Optional[timedelta] = None) -> str:
"""Creates a JWT access token for authentication."""
if expires_delta:
expire = datetime.now(timezone.utc) + expires_delta
else:
expire = datetime.now(timezone.utc) + timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
to_encode = {
"exp": expire,
"sub": str(subject),
"role": role,
"iat": datetime.now(timezone.utc)
}
encoded_jwt = jwt.encode(to_encode, settings.SECRET_KEY, algorithm=settings.ALGORITHM)
return encoded_jwt
def decode_access_token(token: str) -> Optional[dict]:
"""Decodes and validates a JWT token."""
try:
payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM])
return payload
except Exception:
return None
def generate_registration_code() -> str:
"""Generates an agent registration code formatted as OED-XXXX-XXXX."""
part1 = secrets.token_hex(2).upper()
part2 = secrets.token_hex(2).upper()
return f"OED-{part1}-{part2}"
def generate_device_token() -> str:
"""Generates a high-entropy secret token for an agent device."""
return f"oed_sec_{secrets.token_urlsafe(32)}"
def hash_token(token: str) -> str:
"""Returns the SHA-256 hex digest of a token for secure database lookup."""
return hashlib.sha256(token.encode("utf-8")).hexdigest()
+88
View File
@@ -0,0 +1,88 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy import select
from app.core.config import settings
from app.core.database import init_db, AsyncSessionLocal
from app.core.security import get_password_hash
from app.models.models import User
from app.api.auth import router as auth_router
from app.api.clients import router as clients_router
from app.api.jobs import router as jobs_router
from app.api.upload import router as upload_router
from app.api.backups import router as backups_router
from app.api.events import router as events_router
from app.api.stats import router as stats_router
from app.ws.manager import ws_manager
@asynccontextmanager
async def lifespan(app: FastAPI):
# Initialize database tables
await init_db()
# Seed default administrator if not present
async with AsyncSessionLocal() as session:
result = await session.execute(select(User))
admin = result.scalar_one_or_none()
if not admin:
admin_user = User(
email="admin@oneverdrive.local",
hashed_password=get_password_hash("Admin1234!"),
full_name="System Administrator",
role="ADMIN",
is_active=True
)
session.add(admin_user)
await session.commit()
print(">> [OnEver Drive] Default admin created: admin@oneverdrive.local / Admin1234!")
yield
app = FastAPI(
title=settings.PROJECT_NAME,
version=settings.VERSION,
description="Centralized Backup & Sync Platform for Windows on Proxmox VE",
lifespan=lifespan
)
# CORS Middleware to allow Web UI connections
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Register API Routers
app.include_router(auth_router, prefix=settings.API_V1_PREFIX)
app.include_router(clients_router, prefix=settings.API_V1_PREFIX)
app.include_router(jobs_router, prefix=settings.API_V1_PREFIX)
app.include_router(upload_router, prefix=settings.API_V1_PREFIX)
app.include_router(backups_router, prefix=settings.API_V1_PREFIX)
app.include_router(events_router, prefix=settings.API_V1_PREFIX)
app.include_router(stats_router, prefix=settings.API_V1_PREFIX)
@app.websocket("/ws/telemetry")
async def websocket_telemetry(websocket: WebSocket):
"""WebSocket endpoint for real-time dashboard telemetry and live upload meters."""
await ws_manager.connect(websocket)
try:
while True:
# Keep connection open and accept incoming ping/pong or messages
data = await websocket.receive_text()
if data == "ping":
await websocket.send_text("pong")
except WebSocketDisconnect:
ws_manager.disconnect(websocket)
except Exception:
ws_manager.disconnect(websocket)
@app.get("/health")
async def health():
return {
"status": "healthy",
"service": settings.PROJECT_NAME,
"version": settings.VERSION
}
+174
View File
@@ -0,0 +1,174 @@
from datetime import datetime, timezone
import uuid
from sqlalchemy import (
Column, String, Integer, BigInteger, Boolean, DateTime,
ForeignKey, Text, Index
)
from sqlalchemy.orm import relationship
from app.core.database import Base
def utc_now():
return datetime.now(timezone.utc)
class User(Base):
__tablename__ = "users"
id = Column(Integer, primary_key=True, index=True)
email = Column(String(255), unique=True, index=True, nullable=False)
hashed_password = Column(String(255), nullable=False)
full_name = Column(String(255), nullable=True)
role = Column(String(50), default="ADMIN", nullable=False) # ADMIN, OPERATOR, VIEWER
is_active = Column(Boolean, default=True, nullable=False)
created_at = Column(DateTime(timezone=True), default=utc_now, nullable=False)
class Client(Base):
__tablename__ = "clients"
id = Column(Integer, primary_key=True, index=True)
client_code = Column(String(50), unique=True, index=True, nullable=False) # e.g., CLIENT-0001
name = Column(String(255), nullable=False)
hostname = Column(String(255), nullable=True)
os_info = Column(String(255), nullable=True)
ip_address = Column(String(100), nullable=True)
agent_version = Column(String(50), default="1.0.0", nullable=False)
status = Column(String(50), default="OFFLINE", nullable=False) # ONLINE, OFFLINE, SYNCING, ERROR
storage_used_bytes = Column(BigInteger, default=0, nullable=False)
storage_quota_bytes = Column(BigInteger, default=100 * 1024 * 1024 * 1024, nullable=False) # 100 GB default
last_seen_at = Column(DateTime(timezone=True), nullable=True)
last_backup_at = Column(DateTime(timezone=True), nullable=True)
is_active = Column(Boolean, default=True, nullable=False)
created_at = Column(DateTime(timezone=True), default=utc_now, nullable=False)
credentials = relationship("ClientCredential", back_populates="client", cascade="all, delete-orphan")
jobs = relationship("BackupJob", back_populates="client", cascade="all, delete-orphan")
backup_files = relationship("BackupFile", back_populates="client", cascade="all, delete-orphan")
backup_sessions = relationship("BackupSession", back_populates="client", cascade="all, delete-orphan")
class ClientCredential(Base):
__tablename__ = "client_credentials"
id = Column(Integer, primary_key=True, index=True)
client_id = Column(Integer, ForeignKey("clients.id", ondelete="CASCADE"), nullable=False)
device_id = Column(String(100), unique=True, index=True, nullable=False) # Unique UUID
token_hash = Column(String(255), unique=True, index=True, nullable=False)
name = Column(String(255), default="Primary Windows Agent", nullable=False)
is_revoked = Column(Boolean, default=False, nullable=False)
created_at = Column(DateTime(timezone=True), default=utc_now, nullable=False)
last_used_at = Column(DateTime(timezone=True), nullable=True)
client = relationship("Client", back_populates="credentials")
class RegistrationCode(Base):
__tablename__ = "registration_codes"
id = Column(Integer, primary_key=True, index=True)
code = Column(String(50), unique=True, index=True, nullable=False) # e.g., OED-A1B2-C3D4
is_used = Column(Boolean, default=False, nullable=False)
client_name_hint = Column(String(255), nullable=True)
expires_at = Column(DateTime(timezone=True), nullable=False)
created_at = Column(DateTime(timezone=True), default=utc_now, nullable=False)
class BackupJob(Base):
__tablename__ = "backup_jobs"
id = Column(Integer, primary_key=True, index=True)
job_code = Column(String(50), unique=True, index=True, nullable=False) # e.g., JOB-001
client_id = Column(Integer, ForeignKey("clients.id", ondelete="CASCADE"), nullable=False)
name = Column(String(255), nullable=False)
source_path = Column(String(1024), nullable=False) # e.g., C:\SQLBackups
file_patterns = Column(String(255), default="*.bak,*.mdf", nullable=False)
schedule_cron = Column(String(100), default="0 2 * * *", nullable=False) # default 02:00 AM daily
is_active = Column(Boolean, default=True, nullable=False)
# Retention policies
keep_daily = Column(Integer, default=7, nullable=False)
keep_weekly = Column(Integer, default=4, nullable=False)
keep_monthly = Column(Integer, default=12, nullable=False)
min_stable_time_seconds = Column(Integer, default=60, nullable=False)
status = Column(String(50), default="IDLE", nullable=False) # IDLE, RUNNING, ERROR, SUCCESS
last_run_at = Column(DateTime(timezone=True), nullable=True)
next_run_at = Column(DateTime(timezone=True), nullable=True)
created_at = Column(DateTime(timezone=True), default=utc_now, nullable=False)
client = relationship("Client", back_populates="jobs")
backup_files = relationship("BackupFile", back_populates="job", cascade="all, delete-orphan")
backup_sessions = relationship("BackupSession", back_populates="job", cascade="all, delete-orphan")
class BackupSession(Base):
__tablename__ = "backup_sessions"
id = Column(Integer, primary_key=True, index=True)
session_code = Column(String(100), unique=True, index=True, default=lambda: str(uuid.uuid4()), nullable=False)
client_id = Column(Integer, ForeignKey("clients.id", ondelete="CASCADE"), nullable=False)
job_id = Column(Integer, ForeignKey("backup_jobs.id", ondelete="CASCADE"), nullable=True)
filename = Column(String(512), nullable=False)
file_size = Column(BigInteger, nullable=False)
chunk_size = Column(Integer, default=4 * 1024 * 1024, nullable=False)
total_chunks = Column(Integer, nullable=False)
received_chunks_count = Column(Integer, default=0, nullable=False)
sha256_full = Column(String(64), nullable=False)
status = Column(String(50), default="PENDING", nullable=False) # PENDING, UPLOADING, ASSEMBLING, VERIFYING, SUCCESS, FAILED, CANCELLED
error_message = Column(Text, nullable=True)
started_at = Column(DateTime(timezone=True), default=utc_now, nullable=False)
completed_at = Column(DateTime(timezone=True), nullable=True)
client = relationship("Client", back_populates="backup_sessions")
job = relationship("BackupJob", back_populates="backup_sessions")
chunks = relationship("BackupChunk", back_populates="session", cascade="all, delete-orphan")
__table_args__ = (
Index("idx_session_client_status", "client_id", "status"),
)
class BackupChunk(Base):
__tablename__ = "backup_chunks"
id = Column(Integer, primary_key=True, index=True)
session_id = Column(Integer, ForeignKey("backup_sessions.id", ondelete="CASCADE"), nullable=False)
chunk_index = Column(Integer, nullable=False)
chunk_size = Column(Integer, nullable=False)
sha256 = Column(String(64), nullable=False)
is_received = Column(Boolean, default=False, nullable=False)
received_at = Column(DateTime(timezone=True), nullable=True)
session = relationship("BackupSession", back_populates="chunks")
__table_args__ = (
Index("idx_chunk_session_idx", "session_id", "chunk_index", unique=True),
)
class BackupFile(Base):
__tablename__ = "backup_files"
id = Column(Integer, primary_key=True, index=True)
client_id = Column(Integer, ForeignKey("clients.id", ondelete="CASCADE"), nullable=False)
job_id = Column(Integer, ForeignKey("backup_jobs.id", ondelete="CASCADE"), nullable=True)
session_id = Column(Integer, ForeignKey("backup_sessions.id", ondelete="SET NULL"), nullable=True)
filename = Column(String(512), nullable=False)
relative_path = Column(String(1024), nullable=False)
file_size = Column(BigInteger, nullable=False)
sha256 = Column(String(64), nullable=False)
retention_tag = Column(String(50), default="DAILY", nullable=False) # DAILY, WEEKLY, MONTHLY, MANUAL
is_active = Column(Boolean, default=True, nullable=False)
created_at = Column(DateTime(timezone=True), default=utc_now, nullable=False)
client = relationship("Client", back_populates="backup_files")
job = relationship("BackupJob", back_populates="backup_files")
class EventLog(Base):
__tablename__ = "event_logs"
id = Column(Integer, primary_key=True, index=True)
timestamp = Column(DateTime(timezone=True), default=utc_now, nullable=False, index=True)
event_type = Column(String(100), nullable=False, index=True) # LOGIN, CLIENT_REGISTERED, BACKUP_STARTED, BACKUP_COMPLETED, BACKUP_FAILED, etc.
severity = Column(String(50), default="INFO", nullable=False) # INFO, WARNING, ERROR, CRITICAL
client_id = Column(Integer, nullable=True, index=True)
job_id = Column(Integer, nullable=True, index=True)
user_email = Column(String(255), nullable=True)
ip_address = Column(String(100), nullable=True)
message = Column(Text, nullable=False)
details_json = Column(Text, nullable=True)
+207
View File
@@ -0,0 +1,207 @@
from pydantic import BaseModel, Field
from typing import Optional, List, Any, Dict
from datetime import datetime
# --- Auth Schemas ---
class LoginRequest(BaseModel):
email: str
password: str
class TokenResponse(BaseModel):
access_token: str
token_type: str = "bearer"
user: Dict[str, Any]
class UserResponse(BaseModel):
id: int
email: str
full_name: Optional[str]
role: str
is_active: bool
created_at: datetime
model_config = {"from_attributes": True}
# --- Client Schemas ---
class RegistrationCodeCreate(BaseModel):
client_name_hint: Optional[str] = None
expires_in_hours: int = 48
class RegistrationCodeResponse(BaseModel):
code: str
expires_at: datetime
client_name_hint: Optional[str]
class ClientRegisterRequest(BaseModel):
registration_code: str
name: str
hostname: str
os_info: Optional[str] = None
agent_version: str = "1.0.0"
class ClientRegisterResponse(BaseModel):
client_code: str
device_id: str
device_token: str
name: str
server_time: datetime
class ClientHeartbeatRequest(BaseModel):
status: str = "ONLINE" # ONLINE, OFFLINE, SYNCING, ERROR
agent_version: Optional[str] = None
ip_address: Optional[str] = None
class ClientResponse(BaseModel):
id: int
client_code: str
name: str
hostname: Optional[str]
os_info: Optional[str]
ip_address: Optional[str]
agent_version: str
status: str
storage_used_bytes: int
storage_quota_bytes: int
last_seen_at: Optional[datetime]
last_backup_at: Optional[datetime]
is_active: bool
created_at: datetime
model_config = {"from_attributes": True}
# --- Backup Job Schemas ---
class JobCreate(BaseModel):
client_id: int
name: str
source_path: str
file_patterns: str = "*.bak,*.mdf"
schedule_cron: str = "0 2 * * *"
keep_daily: int = 7
keep_weekly: int = 4
keep_monthly: int = 12
min_stable_time_seconds: int = 60
class JobUpdate(BaseModel):
name: Optional[str] = None
source_path: Optional[str] = None
file_patterns: Optional[str] = None
schedule_cron: Optional[str] = None
is_active: Optional[bool] = None
keep_daily: Optional[int] = None
keep_weekly: Optional[int] = None
keep_monthly: Optional[int] = None
min_stable_time_seconds: Optional[int] = None
class JobResponse(BaseModel):
id: int
job_code: str
client_id: int
name: str
source_path: str
file_patterns: str
schedule_cron: str
is_active: bool
keep_daily: int
keep_weekly: int
keep_monthly: int
min_stable_time_seconds: int
status: str
last_run_at: Optional[datetime]
next_run_at: Optional[datetime]
created_at: datetime
model_config = {"from_attributes": True}
# --- Upload Session & Chunk Schemas ---
class UploadSessionInitRequest(BaseModel):
filename: str
file_size: int
sha256: str
chunk_size: int = 4 * 1024 * 1024
job_id: Optional[int] = None
class UploadSessionInitResponse(BaseModel):
session_code: str
filename: str
file_size: int
chunk_size: int
total_chunks: int
received_chunks: List[int]
status: str
class UploadSessionStatusResponse(BaseModel):
session_code: str
filename: str
file_size: int
chunk_size: int
total_chunks: int
received_chunks: List[int]
missing_chunks: List[int]
status: str
progress_percent: float
class ChunkUploadResponse(BaseModel):
chunk_index: int
is_received: bool
total_received: int
total_chunks: int
progress_percent: float
class UploadSessionCompleteResponse(BaseModel):
session_code: str
filename: str
relative_path: str
file_size: int
sha256: str
status: str
completed_at: datetime
# --- Backup File Schemas ---
class BackupFileResponse(BaseModel):
id: int
client_id: int
job_id: Optional[int]
session_id: Optional[int]
filename: str
relative_path: str
file_size: int
sha256: str
retention_tag: str
is_active: bool
created_at: datetime
model_config = {"from_attributes": True}
# --- Event Log Schemas ---
class EventLogResponse(BaseModel):
id: int
timestamp: datetime
event_type: str
severity: str
client_id: Optional[int]
job_id: Optional[int]
user_email: Optional[str]
ip_address: Optional[str]
message: str
details_json: Optional[str]
model_config = {"from_attributes": True}
# --- Dashboard & Storage Stats ---
class StorageStatsResponse(BaseModel):
total_bytes: int
used_bytes: int
free_bytes: int
usage_percent: float
storage_root: str
class DashboardStatsResponse(BaseModel):
total_clients: int
online_clients: int
offline_clients: int
total_jobs: int
backups_today_count: int
backups_today_success: int
backups_today_failed: int
active_uploads_count: int
storage: StorageStatsResponse
+48
View File
@@ -0,0 +1,48 @@
import json
from datetime import datetime, timezone
from typing import Optional, Dict, Any
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.models import EventLog
from app.ws.manager import ws_manager
async def log_event(
db: AsyncSession,
event_type: str,
message: str,
severity: str = "INFO",
client_id: Optional[int] = None,
job_id: Optional[int] = None,
user_email: Optional[str] = None,
ip_address: Optional[str] = None,
details: Optional[Dict[str, Any]] = None
) -> EventLog:
"""Records an audit event in the database and broadcasts it over WebSockets."""
details_str = json.dumps(details, default=str) if details else None
event = EventLog(
timestamp=datetime.now(timezone.utc),
event_type=event_type,
severity=severity,
client_id=client_id,
job_id=job_id,
user_email=user_email,
ip_address=ip_address,
message=message,
details_json=details_str
)
db.add(event)
await db.commit()
await db.refresh(event)
# Broadcast real-time event to all Web UI connections
await ws_manager.broadcast("EVENT_LOG", {
"id": event.id,
"timestamp": event.timestamp.isoformat(),
"event_type": event.event_type,
"severity": event.severity,
"client_id": event.client_id,
"job_id": event.job_id,
"message": event.message
})
return event
+110
View File
@@ -0,0 +1,110 @@
from datetime import datetime, timedelta, timezone
from typing import List, Set
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.models.models import BackupJob, BackupFile, Client
from app.storage.local import storage_provider
from app.services.event_service import log_event
async def apply_retention_policy(db: AsyncSession, job_id: int) -> int:
"""
Applies retention rules (daily, weekly, monthly) for a specific backup job.
Safely deletes obsolete backup files from storage and database.
Returns the number of pruned backup files.
"""
# Fetch job details
result = await db.execute(select(BackupJob).where(BackupJob.id == job_id))
job = result.scalar_one_or_none()
if not job:
return 0
# Fetch all active backup files for this job ordered by creation date descending
result = await db.execute(
select(BackupFile)
.where(BackupFile.job_id == job_id, BackupFile.is_active == True)
.order_by(BackupFile.created_at.desc())
)
backup_files: List[BackupFile] = result.scalars().all()
if not backup_files:
return 0
now = datetime.now(timezone.utc)
protected_file_ids: Set[int] = set()
# Always keep the most recent backup regardless of age to avoid empty repo
protected_file_ids.add(backup_files[0].id)
# 1. Daily retention: Keep 1 backup per calendar day for the last `job.keep_daily` days
seen_days = set()
for bf in backup_files:
day_key = bf.created_at.strftime("%Y-%m-%d")
age_days = (now - bf.created_at).total_seconds() / 86400.0
if age_days <= job.keep_daily:
if day_key not in seen_days:
seen_days.add(day_key)
protected_file_ids.add(bf.id)
# 2. Weekly retention: Keep 1 backup per calendar week for the last `job.keep_weekly` weeks
seen_weeks = set()
for bf in backup_files:
week_key = f"{bf.created_at.year}-W{bf.created_at.isocalendar()[1]:02d}"
age_weeks = (now - bf.created_at).total_seconds() / (86400.0 * 7)
if age_weeks <= job.keep_weekly:
if week_key not in seen_weeks:
seen_weeks.add(week_key)
protected_file_ids.add(bf.id)
# 3. Monthly retention: Keep 1 backup per calendar month for the last `job.keep_monthly` months
seen_months = set()
for bf in backup_files:
month_key = bf.created_at.strftime("%Y-%m")
# Approximate age in months (30 days per month)
age_months = (now - bf.created_at).total_seconds() / (86400.0 * 30.4375)
if age_months <= job.keep_monthly:
if month_key not in seen_months:
seen_months.add(month_key)
protected_file_ids.add(bf.id)
# Identify files to delete
files_to_delete = [bf for bf in backup_files if bf.id not in protected_file_ids]
pruned_count = 0
reclaimed_bytes = 0
for bf in files_to_delete:
try:
# Delete from physical storage
await storage_provider.delete_backup_file(bf.relative_path)
bf.is_active = False
pruned_count += 1
reclaimed_bytes += bf.file_size
except Exception as ex:
# Log failure but continue processing other files
await log_event(
db=db,
event_type="RETENTION_ERROR",
message=f"Failed to delete pruned backup file {bf.filename}: {str(ex)}",
severity="WARNING",
client_id=job.client_id,
job_id=job.id
)
if pruned_count > 0:
# Update client storage used counter
client_res = await db.execute(select(Client).where(Client.id == job.client_id))
client = client_res.scalar_one_or_none()
if client:
client.storage_used_bytes = max(0, client.storage_used_bytes - reclaimed_bytes)
await db.commit()
await log_event(
db=db,
event_type="RETENTION_APPLIED",
message=f"Retention policy applied for job '{job.name}': {pruned_count} obsolete backups removed, {reclaimed_bytes / (1024*1024):.2f} MB reclaimed.",
severity="INFO",
client_id=job.client_id,
job_id=job.id
)
return pruned_count
+277
View File
@@ -0,0 +1,277 @@
import math
from datetime import datetime, timezone
from typing import Tuple, List, Dict, Any, Optional
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.models.models import Client, BackupJob, BackupSession, BackupChunk, BackupFile
from app.storage.local import storage_provider
from app.services.event_service import log_event
from app.services.retention_service import apply_retention_policy
from app.ws.manager import ws_manager
async def create_or_resume_session(
db: AsyncSession,
client: Client,
filename: str,
file_size: int,
sha256_full: str,
chunk_size: int = 4 * 1024 * 1024,
job_id: Optional[int] = None
) -> Tuple[BackupSession, List[int]]:
"""
Initializes a new upload session or resumes an existing incomplete session
for the specified file and hash.
"""
total_chunks = max(1, math.ceil(file_size / chunk_size))
# Check for existing incomplete session for this client and file hash
query = (
select(BackupSession)
.where(
BackupSession.client_id == client.id,
BackupSession.sha256_full == sha256_full,
BackupSession.file_size == file_size,
BackupSession.status.in_(["PENDING", "UPLOADING"])
)
)
result = await db.execute(query)
session = result.scalar_one_or_none()
if session:
# Resume existing session
received_chunks = await storage_provider.get_received_chunks(session.session_code)
session.received_chunks_count = len(received_chunks)
await db.commit()
await db.refresh(session)
return session, received_chunks
# Create new upload session
session = BackupSession(
client_id=client.id,
job_id=job_id,
filename=filename,
file_size=file_size,
chunk_size=chunk_size,
total_chunks=total_chunks,
received_chunks_count=0,
sha256_full=sha256_full,
status="UPLOADING",
started_at=datetime.now(timezone.utc)
)
db.add(session)
await db.commit()
await db.refresh(session)
# Initialize temporary storage
await storage_provider.init_session_storage(session.session_code)
await log_event(
db=db,
event_type="BACKUP_STARTED",
message=f"Upload session initiated for '{filename}' ({file_size / (1024*1024):.2f} MB, {total_chunks} chunks).",
severity="INFO",
client_id=client.id,
job_id=job_id,
details={"session_code": session.session_code, "total_chunks": total_chunks}
)
return session, []
async def process_chunk_upload(
db: AsyncSession,
session: BackupSession,
chunk_index: int,
chunk_data: bytes,
chunk_sha256: Optional[str] = None
) -> Dict[str, Any]:
"""
Saves a chunk to temporary storage, registers chunk in database,
and broadcasts live progress telemetry.
"""
if session.status not in ["PENDING", "UPLOADING"]:
raise ValueError(f"Cannot upload chunk: session is currently in state {session.status}")
if chunk_index < 0 or chunk_index >= session.total_chunks:
raise ValueError(f"Invalid chunk_index {chunk_index}. Session total chunks: {session.total_chunks}")
# Save to storage (performs chunk SHA-256 verification if provided)
await storage_provider.save_chunk(
session_code=session.session_code,
chunk_index=chunk_index,
chunk_data=chunk_data,
expected_sha256=chunk_sha256
)
# Record in database
result = await db.execute(
select(BackupChunk).where(
BackupChunk.session_id == session.id,
BackupChunk.chunk_index == chunk_index
)
)
chunk_rec = result.scalar_one_or_none()
if not chunk_rec:
chunk_rec = BackupChunk(
session_id=session.id,
chunk_index=chunk_index,
chunk_size=len(chunk_data),
sha256=chunk_sha256 or "",
is_received=True,
received_at=datetime.now(timezone.utc)
)
db.add(chunk_rec)
else:
chunk_rec.is_received = True
chunk_rec.received_at = datetime.now(timezone.utc)
# Count received chunks
received_list = await storage_provider.get_received_chunks(session.session_code)
session.received_chunks_count = len(received_list)
await db.commit()
progress_pct = round((session.received_chunks_count / session.total_chunks) * 100, 2)
# Broadcast live telemetry over WebSocket
await ws_manager.broadcast("UPLOAD_PROGRESS", {
"session_code": session.session_code,
"filename": session.filename,
"client_id": session.client_id,
"chunk_index": chunk_index,
"received_chunks": session.received_chunks_count,
"total_chunks": session.total_chunks,
"progress_percent": progress_pct
})
return {
"chunk_index": chunk_index,
"is_received": True,
"total_received": session.received_chunks_count,
"total_chunks": session.total_chunks,
"progress_percent": progress_pct
}
async def get_session_status_info(
db: AsyncSession,
session: BackupSession
) -> Dict[str, Any]:
"""Returns detailed session status and lists of received / missing chunks."""
received = await storage_provider.get_received_chunks(session.session_code)
received_set = set(received)
missing = [i for i in range(session.total_chunks) if i not in received_set]
progress_pct = round((len(received) / session.total_chunks) * 100, 2)
return {
"session_code": session.session_code,
"filename": session.filename,
"file_size": session.file_size,
"chunk_size": session.chunk_size,
"total_chunks": session.total_chunks,
"received_chunks": received,
"missing_chunks": missing,
"status": session.status,
"progress_percent": progress_pct
}
async def complete_session(
db: AsyncSession,
session: BackupSession
) -> BackupFile:
"""
Assembles chunks into final storage, verifies SHA-256 integrity,
updates client stats, applies retention policy, and logs completion.
"""
# Fetch client and job codes for directory naming
client_res = await db.execute(select(Client).where(Client.id == session.client_id))
client = client_res.scalar_one_or_none()
if not client:
raise ValueError(f"Client {session.client_id} not found")
job_code = "DEFAULT"
if session.job_id:
job_res = await db.execute(select(BackupJob).where(BackupJob.id == session.job_id))
job = job_res.scalar_one_or_none()
if job:
job_code = job.job_code
session.status = "ASSEMBLING"
await db.commit()
try:
# Assemble and verify streaming SHA-256
rel_path, final_sha256, total_bytes = await storage_provider.assemble_file(
session_code=session.session_code,
client_code=client.client_code,
job_code=job_code,
filename=session.filename,
total_chunks=session.total_chunks,
expected_sha256=session.sha256_full
)
session.status = "SUCCESS"
session.completed_at = datetime.now(timezone.utc)
# Create BackupFile record
backup_file = BackupFile(
client_id=client.id,
job_id=session.job_id,
session_id=session.id,
filename=session.filename,
relative_path=rel_path,
file_size=total_bytes,
sha256=final_sha256,
retention_tag="DAILY",
is_active=True,
created_at=datetime.now(timezone.utc)
)
db.add(backup_file)
# Update client storage and last backup timestamp
client.storage_used_bytes += total_bytes
client.last_backup_at = datetime.now(timezone.utc)
await db.commit()
await db.refresh(backup_file)
# Log completion event
await log_event(
db=db,
event_type="BACKUP_COMPLETED",
message=f"Backup successfully verified & stored: '{session.filename}' ({total_bytes / (1024*1024):.2f} MB). SHA-256: {final_sha256[:16]}...",
severity="INFO",
client_id=client.id,
job_id=session.job_id,
details={"sha256": final_sha256, "file_size": total_bytes, "path": rel_path}
)
# Apply retention policy if associated with a job
if session.job_id:
await apply_retention_policy(db, session.job_id)
# Broadcast completion
await ws_manager.broadcast("UPLOAD_COMPLETED", {
"session_code": session.session_code,
"filename": session.filename,
"client_id": client.id,
"file_size": total_bytes,
"sha256": final_sha256,
"status": "SUCCESS"
})
return backup_file
except Exception as ex:
session.status = "FAILED"
session.error_message = str(ex)
await db.commit()
await log_event(
db=db,
event_type="BACKUP_FAILED",
message=f"Backup assembly/verification failed for '{session.filename}': {str(ex)}",
severity="ERROR",
client_id=client.id,
job_id=session.job_id,
details={"error": str(ex)}
)
raise ex
+64
View File
@@ -0,0 +1,64 @@
from abc import ABC, abstractmethod
from typing import List, Tuple, Dict, Any, Optional
class BaseStorageProvider(ABC):
"""Abstract interface for OnEver Drive storage backends (Local FS, S3, MinIO, etc.)."""
@abstractmethod
async def init_session_storage(self, session_code: str) -> None:
"""Prepares temporary storage directory for an incoming upload session."""
pass
@abstractmethod
async def save_chunk(
self,
session_code: str,
chunk_index: int,
chunk_data: bytes,
expected_sha256: Optional[str] = None
) -> bool:
"""Saves a single chunk, validates its hash, and returns True on success."""
pass
@abstractmethod
async def get_received_chunks(self, session_code: str) -> List[int]:
"""Returns the list of indices of all successfully stored chunks for a session."""
pass
@abstractmethod
async def assemble_file(
self,
session_code: str,
client_code: str,
job_code: str,
filename: str,
total_chunks: int,
expected_sha256: str
) -> Tuple[str, str, int]:
"""
Assembles all stored chunks in order into the final destination file,
calculates full streaming SHA-256 hash, and verifies integrity.
Returns: (relative_storage_path, actual_sha256, file_size_bytes).
Raises ValueError if integrity check fails or chunks are missing.
"""
pass
@abstractmethod
async def delete_session_temp(self, session_code: str) -> None:
"""Cleans up temporary chunks after assembly or cancellation."""
pass
@abstractmethod
async def delete_backup_file(self, relative_path: str) -> bool:
"""Deletes a backup file from storage."""
pass
@abstractmethod
async def get_file_path(self, relative_path: str) -> str:
"""Resolves absolute path for reading/restoring."""
pass
@abstractmethod
async def get_storage_stats(self) -> Dict[str, Any]:
"""Returns storage capacity stats: {total_bytes, used_bytes, free_bytes, usage_percent}."""
pass
+168
View File
@@ -0,0 +1,168 @@
import os
import shutil
import hashlib
from datetime import datetime, timezone
from pathlib import Path
from typing import List, Tuple, Dict, Any, Optional
import aiofiles
from app.core.config import settings
from app.storage.base import BaseStorageProvider
class LocalStorageProvider(BaseStorageProvider):
"""Local filesystem storage provider designed for Proxmox VE dedicated mount volumes."""
def __init__(self, root_dir: Optional[str] = None, temp_dir: Optional[str] = None):
self.root_dir = Path(root_dir or settings.STORAGE_ROOT).resolve()
self.temp_dir = Path(temp_dir or settings.STORAGE_TEMP_ROOT).resolve()
self.root_dir.mkdir(parents=True, exist_ok=True)
self.temp_dir.mkdir(parents=True, exist_ok=True)
def _get_session_temp_dir(self, session_code: str) -> Path:
return self.temp_dir / session_code
async def init_session_storage(self, session_code: str) -> None:
session_path = self._get_session_temp_dir(session_code)
session_path.mkdir(parents=True, exist_ok=True)
async def save_chunk(
self,
session_code: str,
chunk_index: int,
chunk_data: bytes,
expected_sha256: Optional[str] = None
) -> bool:
# Validate individual chunk hash if provided
if expected_sha256:
actual_chunk_hash = hashlib.sha256(chunk_data).hexdigest()
if actual_chunk_hash.lower() != expected_sha256.lower():
raise ValueError(
f"Chunk {chunk_index} checksum mismatch: expected {expected_sha256}, got {actual_chunk_hash}"
)
session_path = self._get_session_temp_dir(session_code)
session_path.mkdir(parents=True, exist_ok=True)
chunk_file = session_path / f"{chunk_index:08d}.chunk"
async with aiofiles.open(chunk_file, "wb") as f:
await f.write(chunk_data)
return True
async def get_received_chunks(self, session_code: str) -> List[int]:
session_path = self._get_session_temp_dir(session_code)
if not session_path.exists():
return []
chunks = []
for file in session_path.glob("*.chunk"):
try:
index = int(file.stem)
chunks.append(index)
except ValueError:
continue
chunks.sort()
return chunks
async def assemble_file(
self,
session_code: str,
client_code: str,
job_code: str,
filename: str,
total_chunks: int,
expected_sha256: str
) -> Tuple[str, str, int]:
session_path = self._get_session_temp_dir(session_code)
if not session_path.exists():
raise FileNotFoundError(f"Upload session temporary directory {session_code} does not exist")
# Verify all chunks are present
received_chunks = set(await self.get_received_chunks(session_code))
missing_chunks = [i for i in range(total_chunks) if i not in received_chunks]
if missing_chunks:
raise ValueError(f"Cannot assemble file. Missing {len(missing_chunks)} chunks: {missing_chunks[:10]}...")
# Prepare client isolated destination directory
dest_dir = self.root_dir / "clients" / client_code / (job_code or "DEFAULT")
dest_dir.mkdir(parents=True, exist_ok=True)
timestamp_str = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
safe_filename = Path(filename).name
target_filename = f"{timestamp_str}_{safe_filename}"
target_path = dest_dir / target_filename
relative_path = str(target_path.relative_to(self.root_dir)).replace("\\", "/")
hasher = hashlib.sha256()
total_bytes = 0
# Stream and concatenate all chunks in sequential order
async with aiofiles.open(target_path, "wb") as out_file:
for idx in range(total_chunks):
chunk_file = session_path / f"{idx:08d}.chunk"
if not chunk_file.exists():
# Clean up target on failure
if target_path.exists():
target_path.unlink()
raise FileNotFoundError(f"Missing chunk file {chunk_file}")
async with aiofiles.open(chunk_file, "rb") as in_chunk:
while True:
buffer = await in_chunk.read(1024 * 1024) # 1MB buffer
if not buffer:
break
hasher.update(buffer)
total_bytes += len(buffer)
await out_file.write(buffer)
final_sha256 = hasher.hexdigest()
# Strict integrity check against client's pre-calculated full SHA-256
if final_sha256.lower() != expected_sha256.lower():
if target_path.exists():
target_path.unlink()
raise ValueError(
f"Full file integrity check failed! Expected SHA-256: {expected_sha256}, Actual: {final_sha256}"
)
# Cleanup temporary chunks upon confirmed assembly & integrity verification
await self.delete_session_temp(session_code)
return relative_path, final_sha256, total_bytes
async def delete_session_temp(self, session_code: str) -> None:
session_path = self._get_session_temp_dir(session_code)
if session_path.exists():
shutil.rmtree(session_path, ignore_errors=True)
async def delete_backup_file(self, relative_path: str) -> bool:
full_path = (self.root_dir / relative_path).resolve()
# Security guard: prevent path traversal outside root_dir
if not str(full_path).startswith(str(self.root_dir)):
raise PermissionError("Attempted path traversal outside storage root")
if full_path.exists() and full_path.is_file():
full_path.unlink()
return True
return False
async def get_file_path(self, relative_path: str) -> str:
full_path = (self.root_dir / relative_path).resolve()
if not str(full_path).startswith(str(self.root_dir)):
raise PermissionError("Attempted path traversal outside storage root")
if not full_path.exists():
raise FileNotFoundError(f"Backup file {relative_path} not found")
return str(full_path)
async def get_storage_stats(self) -> Dict[str, Any]:
total, used, free = shutil.disk_usage(self.root_dir)
usage_pct = round((used / total) * 100, 2) if total > 0 else 0
return {
"total_bytes": total,
"used_bytes": used,
"free_bytes": free,
"usage_percent": usage_pct,
"storage_root": str(self.root_dir)
}
# Global singleton storage provider
storage_provider = LocalStorageProvider()
+36
View File
@@ -0,0 +1,36 @@
import json
from typing import List, Dict, Any
from fastapi import WebSocket
class WebSocketManager:
"""Manages active WebSocket connections for live telemetry and dashboard updates."""
def __init__(self):
self.active_connections: List[WebSocket] = []
async def connect(self, websocket: WebSocket):
await websocket.accept()
self.active_connections.append(websocket)
def disconnect(self, websocket: WebSocket):
if websocket in self.active_connections:
self.active_connections.remove(websocket)
async def broadcast(self, message_type: str, data: Dict[str, Any]):
"""Broadcasts a structured JSON event to all connected dashboard clients."""
payload = {
"type": message_type,
"data": data
}
message_str = json.dumps(payload, default=str)
dead_connections = []
for connection in self.active_connections:
try:
await connection.send_text(message_str)
except Exception:
dead_connections.append(connection)
for dead in dead_connections:
self.disconnect(dead)
ws_manager = WebSocketManager()