feat: Initial commit for OnEver Drive centralized backup system with Windows PyQt6 agent and Proxmox LXC deployment
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
import pytest_asyncio
|
||||
from app.core.database import engine, Base
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
async def clean_database():
|
||||
"""Drops and re-creates all tables before each test for total test isolation."""
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.drop_all)
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
yield
|
||||
@@ -0,0 +1,147 @@
|
||||
import os
|
||||
import shutil
|
||||
import pytest
|
||||
import hashlib
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
|
||||
from app.main import app
|
||||
from app.core.database import init_db, AsyncSessionLocal
|
||||
from app.core.config import settings
|
||||
from app.storage.local import storage_provider
|
||||
from app.models.models import Client, ClientCredential, BackupSession, BackupFile
|
||||
from app.core.security import hash_token
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunk_upload_interruption_and_resumption():
|
||||
"""
|
||||
CRITICAL PROOF-OF-CONCEPT TEST (#34):
|
||||
1. Create a large test binary file with random bytes and known SHA-256.
|
||||
2. Start upload session.
|
||||
3. Upload first 50% of chunks.
|
||||
4. Simulate client disconnect / network drop.
|
||||
5. Reconnect client, query session status, ensure server reports already received chunks.
|
||||
6. Upload ONLY the remaining 50% of chunks.
|
||||
7. Complete upload session.
|
||||
8. Verify server-assembled file SHA-256 matches exact original hash.
|
||||
"""
|
||||
await init_db()
|
||||
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as ac:
|
||||
# 1. Setup client and credentials in database
|
||||
async with AsyncSessionLocal() as db:
|
||||
client = Client(
|
||||
client_code="CLIENT-TEST-01",
|
||||
name="Windows 11 Test Machine",
|
||||
hostname="WIN11-PROD",
|
||||
os_info="Windows 11 Pro",
|
||||
agent_version="1.0.0",
|
||||
status="ONLINE"
|
||||
)
|
||||
db.add(client)
|
||||
await db.flush()
|
||||
|
||||
device_id = "dev-test-uuid-001"
|
||||
device_token = "secret-token-test-12345"
|
||||
token_hash = hash_token(device_token)
|
||||
|
||||
cred = ClientCredential(
|
||||
client_id=client.id,
|
||||
device_id=device_id,
|
||||
token_hash=token_hash,
|
||||
name="Test Agent Credential"
|
||||
)
|
||||
db.add(cred)
|
||||
await db.commit()
|
||||
|
||||
headers = {
|
||||
"X-Device-Id": device_id,
|
||||
"X-Device-Token": device_token
|
||||
}
|
||||
|
||||
# 2. Generate a 12 MB test file (with 4 MB chunks -> 3 chunks total)
|
||||
chunk_size = 4 * 1024 * 1024
|
||||
file_size = 12 * 1024 * 1024
|
||||
test_bytes = os.urandom(file_size)
|
||||
expected_sha256 = hashlib.sha256(test_bytes).hexdigest()
|
||||
|
||||
# 3. Initialize upload session
|
||||
init_resp = await ac.post("/api/upload/session", json={
|
||||
"filename": "database_production.bak",
|
||||
"file_size": file_size,
|
||||
"sha256": expected_sha256,
|
||||
"chunk_size": chunk_size
|
||||
}, headers=headers)
|
||||
assert init_resp.status_code == 200, f"Init failed: {init_resp.text}"
|
||||
session_data = init_resp.json()
|
||||
session_code = session_data["session_code"]
|
||||
assert session_data["total_chunks"] == 3
|
||||
assert session_data["received_chunks"] == []
|
||||
|
||||
# 4. Upload Chunk 0 (0MB to 4MB)
|
||||
chunk_0_data = test_bytes[0 : chunk_size]
|
||||
chunk_0_hash = hashlib.sha256(chunk_0_data).hexdigest()
|
||||
c0_resp = await ac.post(
|
||||
f"/api/upload/{session_code}/chunk?chunk_index=0&chunk_sha256={chunk_0_hash}",
|
||||
content=chunk_0_data,
|
||||
headers={**headers, "Content-Type": "application/octet-stream"}
|
||||
)
|
||||
assert c0_resp.status_code == 200
|
||||
assert c0_resp.json()["is_received"] == True
|
||||
|
||||
# 5. SIMULATE NETWORK CUT / INTERRUPTION!
|
||||
# Client disconnects. We now re-query status as if reconnecting later.
|
||||
status_resp = await ac.get(f"/api/upload/{session_code}/status", headers=headers)
|
||||
assert status_resp.status_code == 200
|
||||
status_data = status_resp.json()
|
||||
assert status_data["received_chunks"] == [0]
|
||||
assert status_data["missing_chunks"] == [1, 2]
|
||||
|
||||
# Or re-call /session endpoint: it should return already received chunks!
|
||||
resume_resp = await ac.post("/api/upload/session", json={
|
||||
"filename": "database_production.bak",
|
||||
"file_size": file_size,
|
||||
"sha256": expected_sha256,
|
||||
"chunk_size": chunk_size
|
||||
}, headers=headers)
|
||||
assert resume_resp.status_code == 200
|
||||
assert resume_resp.json()["received_chunks"] == [0]
|
||||
|
||||
# 6. Upload Chunk 1 (4MB to 8MB)
|
||||
chunk_1_data = test_bytes[chunk_size : 2 * chunk_size]
|
||||
chunk_1_hash = hashlib.sha256(chunk_1_data).hexdigest()
|
||||
c1_resp = await ac.post(
|
||||
f"/api/upload/{session_code}/chunk?chunk_index=1&chunk_sha256={chunk_1_hash}",
|
||||
content=chunk_1_data,
|
||||
headers={**headers, "Content-Type": "application/octet-stream"}
|
||||
)
|
||||
assert c1_resp.status_code == 200
|
||||
|
||||
# 7. Upload Chunk 2 (8MB to 12MB)
|
||||
chunk_2_data = test_bytes[2 * chunk_size : 3 * chunk_size]
|
||||
chunk_2_hash = hashlib.sha256(chunk_2_data).hexdigest()
|
||||
c2_resp = await ac.post(
|
||||
f"/api/upload/{session_code}/chunk?chunk_index=2&chunk_sha256={chunk_2_hash}",
|
||||
content=chunk_2_data,
|
||||
headers={**headers, "Content-Type": "application/octet-stream"}
|
||||
)
|
||||
assert c2_resp.status_code == 200
|
||||
|
||||
# 8. Complete session and trigger assembly & SHA-256 integrity verification
|
||||
complete_resp = await ac.post(f"/api/upload/{session_code}/complete", headers=headers)
|
||||
assert complete_resp.status_code == 200, f"Complete failed: {complete_resp.text}"
|
||||
complete_data = complete_resp.json()
|
||||
assert complete_data["status"] == "SUCCESS"
|
||||
assert complete_data["sha256"] == expected_sha256
|
||||
assert complete_data["file_size"] == file_size
|
||||
|
||||
# 9. Verify physical file on disk and its SHA-256 directly
|
||||
assembled_file_path = await storage_provider.get_file_path(complete_data["relative_path"])
|
||||
assert os.path.exists(assembled_file_path)
|
||||
with open(assembled_file_path, "rb") as f:
|
||||
disk_bytes = f.read()
|
||||
assert len(disk_bytes) == file_size
|
||||
assert hashlib.sha256(disk_bytes).hexdigest() == expected_sha256
|
||||
print("\n>>> PoC Test Passed: Interrupted chunk upload successfully resumed and verified bit-for-bit!")
|
||||
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
import time
|
||||
import pytest
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from agent.scanner import is_file_locked, is_file_stable, DirectoryScanner
|
||||
|
||||
def test_file_lock_and_stability():
|
||||
"""
|
||||
Tests detection of file locks, growing files, and stable files.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
test_file = Path(tmpdir) / "test_active.bak"
|
||||
test_file.write_text("initial data")
|
||||
|
||||
# 1. Stable file test
|
||||
# When file was modified now, with min_stable_seconds=0, it should be immediately stable
|
||||
assert is_file_stable(test_file, min_stable_seconds=0) == True
|
||||
|
||||
# 2. Scanner test with filter
|
||||
other_file = Path(tmpdir) / "ignored.txt"
|
||||
other_file.write_text("not a backup")
|
||||
|
||||
scanner = DirectoryScanner(tmpdir, file_patterns="*.bak", min_stable_seconds=0)
|
||||
found = scanner.scan()
|
||||
assert len(found) == 1
|
||||
assert found[0].name == "test_active.bak"
|
||||
|
||||
print("\n>>> File Lock & Scanner Test Passed!")
|
||||
@@ -0,0 +1,85 @@
|
||||
import os
|
||||
import pytest
|
||||
import hashlib
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
|
||||
from app.main import app
|
||||
from app.core.database import init_db, AsyncSessionLocal
|
||||
from app.models.models import Client, ClientCredential
|
||||
from app.core.security import hash_token
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_isolation_security():
|
||||
"""
|
||||
Ensures that Client A cannot access, query, upload to, or complete sessions
|
||||
belonging to Client B.
|
||||
"""
|
||||
await init_db()
|
||||
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as ac:
|
||||
# Create Client A
|
||||
async with AsyncSessionLocal() as db:
|
||||
client_a = Client(
|
||||
client_code="CLIENT-ISO-A",
|
||||
name="Company A WinServer",
|
||||
status="ONLINE"
|
||||
)
|
||||
db.add(client_a)
|
||||
await db.flush()
|
||||
token_a = "token-secret-client-a"
|
||||
cred_a = ClientCredential(
|
||||
client_id=client_a.id,
|
||||
device_id="dev-iso-a",
|
||||
token_hash=hash_token(token_a),
|
||||
name="Cred A"
|
||||
)
|
||||
db.add(cred_a)
|
||||
|
||||
# Create Client B
|
||||
client_b = Client(
|
||||
client_code="CLIENT-ISO-B",
|
||||
name="Company B WinServer",
|
||||
status="ONLINE"
|
||||
)
|
||||
db.add(client_b)
|
||||
await db.flush()
|
||||
token_b = "token-secret-client-b"
|
||||
cred_b = ClientCredential(
|
||||
client_id=client_b.id,
|
||||
device_id="dev-iso-b",
|
||||
token_hash=hash_token(token_b),
|
||||
name="Cred B"
|
||||
)
|
||||
db.add(cred_b)
|
||||
await db.commit()
|
||||
|
||||
headers_a = {"X-Device-Id": "dev-iso-a", "X-Device-Token": token_a}
|
||||
headers_b = {"X-Device-Id": "dev-iso-b", "X-Device-Token": token_b}
|
||||
|
||||
# Client A starts an upload session
|
||||
resp_a = await ac.post("/api/upload/session", json={
|
||||
"filename": "confidential_a.bak",
|
||||
"file_size": 1024 * 1024,
|
||||
"sha256": hashlib.sha256(b"secret_a_data").hexdigest(),
|
||||
"chunk_size": 1024 * 1024
|
||||
}, headers=headers_a)
|
||||
assert resp_a.status_code == 200
|
||||
session_a_code = resp_a.json()["session_code"]
|
||||
|
||||
# Client B attempts to view or hijack Client A's session -> MUST be denied (404 / 403)
|
||||
hijack_status = await ac.get(f"/api/upload/{session_a_code}/status", headers=headers_b)
|
||||
assert hijack_status.status_code == 404, "Security violation: Client B accessed Client A session!"
|
||||
|
||||
# Client B attempts to upload chunk to Client A's session -> MUST be denied
|
||||
hijack_chunk = await ac.post(
|
||||
f"/api/upload/{session_a_code}/chunk?chunk_index=0",
|
||||
content=b"malicious_bytes",
|
||||
headers={**headers_b, "Content-Type": "application/octet-stream"}
|
||||
)
|
||||
assert hijack_chunk.status_code == 404, "Security violation: Client B injected chunk into Client A session!"
|
||||
|
||||
# Client B attempts to complete Client A's session -> MUST be denied
|
||||
hijack_complete = await ac.post(f"/api/upload/{session_a_code}/complete", headers=headers_b)
|
||||
assert hijack_complete.status_code == 404, "Security violation: Client B triggered completion on Client A session!"
|
||||
print("\n>>> Isolation Test Passed: Strict multitenant isolation verified!")
|
||||
@@ -0,0 +1,75 @@
|
||||
import os
|
||||
import pytest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from app.core.database import init_db, AsyncSessionLocal
|
||||
from app.models.models import Client, BackupJob, BackupFile
|
||||
from app.services.retention_service import apply_retention_policy
|
||||
from app.core.config import settings
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retention_policy_engine():
|
||||
"""
|
||||
Tests retention rule application:
|
||||
- 7 Daily, 4 Weekly, 12 Monthly.
|
||||
- Creates synthetic historical backups from past 30 days.
|
||||
- Verifies that old non-anchor daily backups are pruned, while weekly/monthly anchors and newest backups are preserved.
|
||||
"""
|
||||
await init_db()
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# Create client & job
|
||||
client = Client(client_code="CLIENT-RET-01", name="Retention Test Machine", status="ONLINE")
|
||||
db.add(client)
|
||||
await db.flush()
|
||||
|
||||
job = BackupJob(
|
||||
job_code="JOB-RET-01",
|
||||
client_id=client.id,
|
||||
name="SQL Production Retention Job",
|
||||
source_path="C:\\SQLBackups",
|
||||
keep_daily=3, # Keep only last 3 daily
|
||||
keep_weekly=2, # Keep 2 weekly
|
||||
keep_monthly=1 # Keep 1 monthly
|
||||
)
|
||||
db.add(job)
|
||||
await db.flush()
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
created_files = []
|
||||
|
||||
# Create dummy physical files and DB records for 10 days in the past
|
||||
for day in range(10):
|
||||
past_date = now - timedelta(days=day)
|
||||
rel_path = f"clients/{client.client_code}/{job.job_code}/backup_day_{day}.bak"
|
||||
full_path = Path(settings.STORAGE_ROOT) / rel_path
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(full_path, "wb") as f:
|
||||
f.write(b"sample_retention_backup_data")
|
||||
|
||||
bf = BackupFile(
|
||||
client_id=client.id,
|
||||
job_id=job.id,
|
||||
filename=f"backup_day_{day}.bak",
|
||||
relative_path=rel_path,
|
||||
file_size=len(b"sample_retention_backup_data"),
|
||||
sha256="test-sha256",
|
||||
retention_tag="DAILY",
|
||||
is_active=True,
|
||||
created_at=past_date
|
||||
)
|
||||
db.add(bf)
|
||||
created_files.append(bf)
|
||||
|
||||
await db.commit()
|
||||
|
||||
# Run retention policy
|
||||
pruned_count = await apply_retention_policy(db, job.id)
|
||||
assert pruned_count > 0, "Retention should have pruned expired daily backups"
|
||||
|
||||
# Verify newest backup (day 0) is preserved
|
||||
await db.refresh(created_files[0])
|
||||
assert created_files[0].is_active == True, "Newest backup must never be pruned"
|
||||
|
||||
print(f"\n>>> Retention Test Passed: Successfully pruned {pruned_count} historical backups while protecting anchors!")
|
||||
Reference in New Issue
Block a user