Files
admin-edu-space/backend/app/services/sheets_importer.py
T

643 lines
27 KiB
Python

import urllib.request
import csv
import io
import re
import os
from datetime import datetime, date, time, timedelta
from app import db
from app.models.classroom import Classroom
from app.models.career import Career
from app.models.subject import Subject, Commission
from app.models.reservation import Reservation, ReservationStatus
from app.models.user import User
BASE_CSV_URL = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vSc_T_BQjbn3uPelioCgx52UM5Py-qNhJN0TYPd1kmsN5jdb3Q8rAaIvNMF_2ZTzQt6bH--yIWQKrKR/pub?output=csv'
# Mapeo canónico de carreras de UniCABA para desglosar y normalizar registros compuestos
CANONICAL_CAREERS = {
'DATOS': 'DATOS (CCC)',
'DATOS (CCC)': 'DATOS (CCC)',
'DATOS(CCC)': 'DATOS (CCC)',
'TEC.DIG': 'TEC. DIG. (CCC)',
'TEC. DIG (CCC)': 'TEC. DIG. (CCC)',
'TEC. DIG. (CCC)': 'TEC. DIG. (CCC)',
'TEC DIG': 'TEC. DIG. (CCC)',
'VIDEOJUEGOS': 'VIDEOJUEGOS (CCC)',
'VIDEOJUEGOS (CCC)': 'VIDEOJUEGOS (CCC)',
'PROF PROF': 'PROF PROF (CCC)',
'PROF PROF (CCC)': 'PROF PROF (CCC)',
'CIENCIAS': 'CIENCIAS',
'EF. SALUD': 'EF. SALUD',
'EF. GESTIÓN': 'EF. GESTIÓN',
'EF. GESTION': 'EF. GESTIÓN',
'ETD': 'ETD (CCC)',
'ETD (CCC)': 'ETD (CCC)',
'MONO': 'MONODISCIPLINARES',
'MONODISCIPLINARES': 'MONODISCIPLINARES',
'PUEP': 'PUEP',
'PUEP CCC': 'PUEP CCC',
'INGLÉS': 'INGLÉS',
'INGLES': 'INGLÉS',
'GESTIÓN (CCC)': 'Gestión (CCC)',
'GESTION (CCC)': 'Gestión (CCC)',
'DIPLO. ALFABETIZACIÓN': 'DIPLO. ALFABETIZACIÓN',
'DIPLO. ALFABETIZACION': 'DIPLO. ALFABETIZACIÓN',
'DIPLO. BILINGUE': 'DIPLO. BILINGUE',
'POSG. PRIMARIA': 'POSG. PRIMARIA',
'POSG. INICIAL': 'POSG. INICIAL',
'POSG. INCLUSIVAS': 'POSG. INCLUSIVAS',
}
def map_day_to_idx(day_str):
"""Mapea el nombre del día (o prefijo) a su índice numérico 0..6 (Lunes=0)"""
d = (day_str or '').lower()
if 'lun' in d: return 0
if 'mar' in d: return 1
if 'mie' in d or 'mié' in d: return 2
if 'jue' in d: return 3
if 'vie' in d: return 4
if 'sab' in d or 'sáb' in d: return 5
if 'dom' in d: return 6
return 0
def parse_careers_list(c_str):
"""Parsea y normaliza cadenas de carreras compuestas separadas por guiones o comas"""
if not c_str:
return ['General']
parts = re.split(r'\s*-\s*|(?<=\))\s*-\s*|\s*,\s*', c_str)
result = []
for p in parts:
clean = p.strip()
if not clean:
continue
upper_clean = clean.upper()
matched = False
for k, v in CANONICAL_CAREERS.items():
if upper_clean == k.upper():
if v not in result:
result.append(v)
matched = True
break
if not matched:
if clean not in result:
result.append(clean)
return result if result else ['General']
def parse_subject_codes(code_str):
"""Parsea códigos de materias que pueden venir combinados con barras (ej: ASIG00113 / ASIG000082)"""
if not code_str:
return []
parts = re.split(r'\s*/\s*|\s*,\s*', code_str)
return [p.strip() for p in parts if p.strip()]
def normalize_sheets_url(url):
"""
Normaliza enlaces de Google Sheets (compartidos, de edición, con /u/1/, #gid=... o /pubhtml)
para obtener la URL base que acepta parámetro gid=<gid>&output=csv o format=csv.
"""
if not url:
return BASE_CSV_URL
url = url.strip()
# 1. Enlace publicado web (/d/e/2PACX-... con o sin /u/X/ y con pub o pubhtml)
pub_match = re.search(r'/spreadsheets/(?:u/\d+/)?d/e/([a-zA-Z0-9-_]+)', url)
if pub_match:
doc_id = pub_match.group(1)
return f"https://docs.google.com/spreadsheets/d/e/{doc_id}/pub?output=csv"
# 2. Enlace de edición o exportación (/d/<SPREADSHEET_ID>/...)
sheet_id_match = re.search(r'/spreadsheets/(?:u/\d+/)?d/([a-zA-Z0-9-_]+)', url)
if sheet_id_match:
sheet_id = sheet_id_match.group(1)
return f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv"
return url
SHEETS_CONFIG = [
{ 'name': 'Lunes', 'gid': '1915752353', 'day_idx': 0 },
{ 'name': 'Martes', 'gid': '466778263', 'day_idx': 1 },
{ 'name': 'Miércoles', 'gid': '615483345', 'day_idx': 2 },
{ 'name': 'Jueves', 'gid': '682416571', 'day_idx': 3 },
{ 'name': 'Viernes', 'gid': '1162906711', 'day_idx': 4 },
{ 'name': 'Sábado', 'gid': '752305601', 'day_idx': 5 },
]
TIME_REGEX = re.compile(r'(\d{1,2})(?::(\d{2}))?\s*(?:a|-|to)\s*(\d{1,2})(?::(\d{2}))?', re.IGNORECASE)
class GoogleSheetsImporter:
"""
Servicio para descargar, parsear y sincronizar cursadas académicas,
aulas, carreras y asignaturas compartidas desde el Google Sheet oficial de UniCABA.
"""
def __init__(self, base_url=None):
if base_url:
self.base_url = normalize_sheets_url(base_url)
else:
configured_url = None
try:
from app.models.setting import SystemSetting
configured_url = SystemSetting.get_value('google_sheets_url')
except Exception:
pass
if not configured_url:
configured_url = os.getenv('GOOGLE_SHEETS_URL')
self.base_url = normalize_sheets_url(configured_url) if configured_url else BASE_CSV_URL
def get_sheets_config(self):
"""
Descubre las pestañas y GIDs dinámicamente desde /pubhtml si está disponible,
o utiliza el fallback SHEETS_CONFIG preconfigurado para los 6 días de cursada.
"""
pub_match = re.search(r'/spreadsheets/d/e/([a-zA-Z0-9-_]+)', self.base_url)
if pub_match:
doc_id = pub_match.group(1)
pubhtml_url = f"https://docs.google.com/spreadsheets/d/e/{doc_id}/pubhtml"
try:
req = urllib.request.Request(
pubhtml_url,
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) EduSpace/3.4'}
)
with urllib.request.urlopen(req, timeout=12) as resp:
html_content = resp.read().decode('utf-8', errors='replace')
# Extraer pares name y gid de Google Sheets pubhtml
found = re.findall(r'name:\s*["\']([^"\']+)["\'][^}]*?gid:\s*["\'](\d+)["\']', html_content)
if found:
discovered = []
for name, gid in found:
clean_name = re.sub(r'[^a-zA-ZáéíóúÁÉÍÓÚñÑ]', '', name).capitalize()
d_idx = map_day_to_idx(clean_name)
discovered.append({
'name': clean_name if clean_name else name,
'gid': gid,
'day_idx': d_idx
})
if discovered:
return discovered
except Exception:
pass
return SHEETS_CONFIG
def fetch_sheet_csv(self, gid):
"""Fetch CSV string for a specific sheet gid"""
separator = '&' if '?' in self.base_url else '?'
url = f"{self.base_url}{separator}gid={gid}"
if not (url.startswith('https://') or url.startswith('http://')):
raise ValueError(f"URL scheme not permitted: {url}")
req = urllib.request.Request(
url,
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) EduSpace/3.4'}
)
with urllib.request.urlopen(req, timeout=25) as resp: # nosec B310
return resp.read().decode('utf-8', errors='replace')
def parse_sheet_rows(self, csv_content, sheet_name):
"""Parse raw CSV text into a structured list of class records"""
reader = csv.reader(io.StringIO(csv_content))
rows = list(reader)
header_idx = -1
col_offset = 0
for r_idx, row in enumerate(rows):
for c_idx, cell in enumerate(row):
cell_clean = cell.lower().strip()
if 'codigo' in cell_clean or 'código' in cell_clean:
header_idx = r_idx
col_offset = c_idx
break
if header_idx != -1:
break
if header_idx == -1:
return []
records = []
current_shift = 'Mañana'
for row in rows[header_idx + 1:]:
if len(row) >= col_offset + 5:
def _clean(val):
return val.replace('\ufb01', 'fi').replace('\ufb02', 'fl').strip()
raw_cell = _clean(row[col_offset]).replace('\n', ' - ')
row_full_text = ' '.join(row).upper()
# Detect shift delimiters
if 'TURNO MAÑANA' in row_full_text or 'TURNO MANANA' in row_full_text:
current_shift = 'Mañana'
continue
elif 'TURNO VESPERTINO' in row_full_text or 'TURNO TARDE' in row_full_text or 'TURNO NOCHE' in row_full_text:
current_shift = 'Tarde'
continue
elif raw_cell.lower().startswith('turno'):
if 'mañ' in raw_cell.lower() or 'man' in raw_cell.lower():
current_shift = 'Mañana'
else:
current_shift = 'Tarde'
continue
codigo = raw_cell
asignatura = _clean(row[col_offset + 1]).replace('\n', ' ')
aula = _clean(row[col_offset + 2]).replace('\n', ' ')
carrera = _clean(row[col_offset + 3]).replace('\n', ' ')
horario = _clean(row[col_offset + 4]).replace('\n', ' ')
# Check valid data row
if codigo and asignatura and not 'codigo' in codigo.lower():
# Double-check shift from start hour if possible
record_shift = current_shift
time_match = re.search(r'(\d{1,2}):(\d{2})', horario)
if time_match:
h = int(time_match.group(1))
if h >= 14:
record_shift = 'Tarde'
elif h < 13:
record_shift = 'Mañana'
records.append({
'dia': sheet_name,
'codigo': codigo,
'asignatura': asignatura,
'aula': aula,
'carrera': carrera,
'horario': horario,
'turno': record_shift
})
return records
def normalize_room(self, aula_raw):
"""
Normaliza cadena de aula en (building, room_number, floor, capacity)
"""
aula_upper = aula_raw.upper().strip()
if not aula_upper or any(x in aula_upper for x in ['FERIADO', 'SIN ACTIVIDAD', 'CONSULTAR EN BEDELIA']):
return None
if 'VIRTUAL' in aula_upper:
return {
'building': 'Campus Virtual',
'code': 'VIRTUAL',
'floor': 0,
'capacity': 0,
'description': 'Espacio áulico virtual UniCABA (Capacidad Ilimitada ∞)'
}
if 'AUDITORIO' in aula_upper:
return {
'building': 'Edificio Central',
'code': 'Auditorio',
'floor': 0,
'capacity': 150,
'description': 'Auditorio principal UniCABA'
}
if 'SALA DE REUNI' in aula_upper:
floor = 2 if ('2DO' in aula_upper or '2°' in aula_upper) else 4
return {
'building': 'Edificio Central',
'code': f'Sala Reunión {floor}°P',
'floor': floor,
'capacity': 15,
'description': f'Sala de reunión piso {floor}'
}
# Standard classrooms: "Aula 402", "AULA 203", "306 - 406"
num_match = re.search(r'\d+', aula_raw)
if num_match:
num = num_match.group(0)
floor = int(num[0]) if len(num) >= 3 else 1
return {
'building': 'Edificio Central',
'code': f'Aula {num}',
'floor': floor,
'capacity': 35,
'description': f'Aula física {num} - Piso {floor}'
}
# Generic fallback
return {
'building': 'Edificio Central',
'code': aula_raw,
'floor': 1,
'capacity': 30,
'description': aula_raw
}
def parse_time_range(self, horario_str):
"""Parse '8:00 a 9:00hs' into (time(8, 0), time(9, 0))"""
m = TIME_REGEX.search(horario_str)
if not m:
return None, None
try:
h1 = int(m.group(1))
m1 = int(m.group(2) or 0)
h2 = int(m.group(3))
m2 = int(m.group(4) or 0)
return time(h1, m1), time(h2, m2)
except Exception:
return None, None
def sync(self):
"""
Proceso principal de sincronización:
1. Descarga las 6 hojas de cálculo semanales (Lunes a Sábado).
2. Normaliza carreras individuales y asignaturas compartidas (Many-to-Many).
3. Upserts Aulas, Carreras, Asignaturas, Comisiones.
4. Genera Reservas en el calendario para el semestre académico.
"""
stats = {
'total_rows_processed': 0,
'careers_created': 0,
'careers_updated': 0,
'classrooms_created': 0,
'classrooms_updated': 0,
'subjects_created': 0,
'subjects_updated': 0,
'commissions_created': 0,
'reservations_created': 0,
'by_day': {},
'errors': [],
'sample_records': []
}
all_raw_records = []
sheets_to_process = self.get_sheets_config()
for sheet_cfg in sheets_to_process:
day_name = sheet_cfg['name']
try:
csv_data = self.fetch_sheet_csv(sheet_cfg['gid'])
rows = self.parse_sheet_rows(csv_data, day_name)
all_raw_records.extend([(r, sheet_cfg['day_idx']) for r in rows])
stats['by_day'][day_name] = {
'gid': sheet_cfg['gid'],
'rows_count': len(rows),
'status': 'OK'
}
except Exception as e:
err_msg = f"Error al descargar o procesar la hoja de {day_name}: {str(e)}"
stats['errors'].append(err_msg)
stats['by_day'][day_name] = {
'gid': sheet_cfg['gid'],
'rows_count': 0,
'status': 'Error',
'error': str(e)
}
stats['total_rows_processed'] = len(all_raw_records)
if not all_raw_records:
return stats
# Default admin user for reservations
default_user = User.query.filter_by(role='ADMIN').first() or User.query.first()
default_user_id = default_user.id if default_user else 1
# Ensure institutional buildings exist with official address
from app.models.building import Building
b_central = Building.query.filter_by(name='Edificio Central').first()
if not b_central:
b_central = Building(
name='Edificio Central',
code='CENTRAL',
address='Tte. Gral. Juan Domingo Perón 802, CABA',
floors=4,
description='Sede central institucional de UniCABA. Aulas de grado, auditorio y laboratorios.'
)
db.session.add(b_central)
db.session.flush()
else:
b_central.address = 'Tte. Gral. Juan Domingo Perón 802, CABA'
b_virtual = Building.query.filter_by(name='Campus Virtual').first()
if not b_virtual:
b_virtual = Building(
name='Campus Virtual',
code='VIRTUAL',
address='Entorno Digital / Microsoft Teams - Moodle',
floors=0,
description='Entorno de aulas virtuales y clases a distancia'
)
db.session.add(b_virtual)
db.session.flush()
buildings_cache = {b.name.strip().upper(): b for b in Building.query.all()}
careers_cache = {c.name.strip().upper(): c for c in Career.query.all()}
classrooms_cache = {(c.building.strip().upper(), c.code.strip().upper()): c for c in Classroom.query.all()}
subjects_cache = {s.code.strip().upper(): s for s in Subject.query.all()}
# Calculate current semester dates (anchored on academic year 2026)
base_monday = date(2026, 8, 31)
weeks_to_generate = 16 # Generar reservas para 16 semanas (cuatrimestre completo Ago - Dic 2026)
for rec, day_idx in all_raw_records:
try:
codigo = rec['codigo']
asignatura = rec['asignatura']
aula_raw = rec['aula']
carrera_raw = rec['carrera'].strip()
horario_raw = rec['horario']
dia = rec['dia']
turno = rec.get('turno', 'Mañana')
# 1. Upsert Careers (Desglosar carreras múltiples normalizadas)
carreras_list = parse_careers_list(carrera_raw)
career_objs = []
for c_name in carreras_list:
c_key = c_name.upper()
career = careers_cache.get(c_key)
if not career:
career = Career.query.filter(Career.name.ilike(c_name)).first()
if not career:
career = Career(name=c_name)
db.session.add(career)
db.session.flush()
stats['careers_created'] += 1
careers_cache[c_key] = career
career_objs.append(career)
primary_career = career_objs[0] if career_objs else None
# 2. Upsert Classroom
room_info = self.normalize_room(aula_raw)
classroom = None
if room_info:
room_key = (room_info['building'].upper(), room_info['code'].upper())
classroom = classrooms_cache.get(room_key)
b_entity = buildings_cache.get(room_info['building'].upper())
if not classroom:
classroom = Classroom(
building=room_info['building'],
building_id=b_entity.id if b_entity else None,
floor=room_info['floor'],
capacity=room_info['capacity'],
description=room_info['description'],
is_active=True
)
classroom.code = room_info['code']
db.session.add(classroom)
db.session.flush()
classrooms_cache[room_key] = classroom
stats['classrooms_created'] += 1
else:
classroom.capacity = max(classroom.capacity, room_info['capacity'])
if not classroom.building_id and b_entity:
classroom.building_id = b_entity.id
# 3. Upsert Subject (Soporte Many-to-Many con múltiples carreras y subcódigos)
subject_key = codigo.upper()
sub_codes = parse_subject_codes(codigo)
subject = subjects_cache.get(subject_key)
if not subject:
for sc in sub_codes:
if sc.upper() in subjects_cache:
subject = subjects_cache[sc.upper()]
break
if not subject:
subject = Subject.query.filter_by(code=codigo).first()
if not subject:
for sc in sub_codes:
subj_by_sc = Subject.query.filter_by(code=sc).first()
if subj_by_sc:
subject = subj_by_sc
break
department_str = ", ".join([c.name for c in career_objs]) if career_objs else (primary_career.name if primary_career else 'General')
if not subject:
subject = Subject(
code=codigo,
name=asignatura,
career_id=primary_career.id if primary_career else None,
department=department_str,
credits=4,
is_active=True
)
subject.careers = career_objs
db.session.add(subject)
db.session.flush()
stats['subjects_created'] += 1
else:
if subject.name != asignatura:
subject.name = asignatura
stats['subjects_updated'] += 1
# Vincular todas las carreras a la asignatura sin duplicados
for c_obj in career_objs:
if c_obj not in subject.careers:
subject.careers.append(c_obj)
if not subject.career_id and primary_career:
subject.career_id = primary_career.id
subject.department = ", ".join([c.name for c in subject.careers]) if subject.careers else department_str
# Registrar en caché con código completo y con subcódigos individuales
subjects_cache[subject_key] = subject
for sc in sub_codes:
subjects_cache[sc.upper()] = subject
# 4. Upsert Commission
comm_code = 'C1'
if '-E' in codigo:
comm_code = codigo.split('-')[-1]
elif '-0' in codigo:
comm_code = 'C' + codigo.split('-')[-1]
commission = Commission.query.filter_by(
subject_id=subject.id,
code=comm_code,
semester='2026-2',
year=2026
).first()
if not commission:
commission = Commission(
subject_id=subject.id,
code=comm_code,
semester='2026-2',
year=2026,
teacher_id=default_user_id,
max_students=classroom.capacity if classroom else 35,
current_students=0,
schedule=f"{dia} {horario_raw}",
shift=turno,
active=True
)
db.session.add(commission)
db.session.flush()
stats['commissions_created'] += 1
else:
commission.schedule = f"{dia} {horario_raw}"
commission.shift = turno
if classroom:
commission.max_students = max(commission.max_students or 0, classroom.capacity)
# 5. Generar Reservas de Calendario
t_start, t_end = self.parse_time_range(horario_raw)
if classroom and t_start and t_end and day_idx is not None:
is_room_virtual = bool(classroom.is_virtual or 'VIRTUAL' in (classroom.code or '').upper())
v_link = None
if is_room_virtual:
clean_subj = re.sub(r'[^a-zA-Z0-9]', '', subject.code or 'cls').lower()
clean_comm = re.sub(r'[^a-zA-Z0-9]', '', commission.code or 'c1').lower()
v_link = f"https://meet.google.com/edu-{clean_subj}-{clean_comm}"
if not commission.virtual_link:
commission.virtual_link = v_link
for w in range(weeks_to_generate):
class_date = base_monday + timedelta(days=day_idx, weeks=w)
dt_start = datetime.combine(class_date, t_start)
dt_end = datetime.combine(class_date, t_end)
if is_room_virtual:
existing_res = Reservation.query.filter_by(
classroom_id=classroom.id,
commission_id=commission.id,
start_time=dt_start,
end_time=dt_end
).first()
else:
existing_res = Reservation.query.filter_by(
classroom_id=classroom.id,
start_time=dt_start,
end_time=dt_end
).first()
c_desc = ", ".join([c.name for c in career_objs]) if career_objs else 'General'
if not existing_res:
res = Reservation(
classroom_id=classroom.id,
commission_id=commission.id,
user_id=default_user_id,
start_time=dt_start,
end_time=dt_end,
purpose=f"{subject.code} - {subject.name}",
status='CONFIRMED',
expected_attendees=min(classroom.capacity, 30) if not is_room_virtual else 35,
shift=turno,
virtual_link=v_link,
notes=f"Clase semanal {dia} ({turno}) - Carreras: {c_desc}"
)
db.session.add(res)
stats['reservations_created'] += 1
else:
existing_res.shift = turno
existing_res.notes = f"Clase semanal {dia} ({turno}) - Carreras: {c_desc}"
if is_room_virtual and not existing_res.virtual_link:
existing_res.virtual_link = v_link
if len(stats['sample_records']) < 15:
stats['sample_records'].append({
'dia': dia,
'codigo': codigo,
'asignatura': asignatura,
'carreras': [c.name for c in career_objs],
'aula': classroom.code_display if classroom else aula_raw,
'horario': horario_raw
})
except Exception as row_err:
stats['errors'].append(f"Error en fila '{rec.get('codigo', '?')}': {str(row_err)}")
db.session.commit()
return stats