255 lines
10 KiB
Python
255 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""sync_forms_brand.py
|
|
|
|
Sincroniza el catalogo de formularios y todos los submissions de la cuenta de
|
|
Marca Principal (GbKkBpCmKu2QmloKFHy3) hacia la base de datos local
|
|
(mp_manager.sqlite). El audit_brand_sucursal_vs_form.py consume estos datos.
|
|
|
|
Por que solo Marca:
|
|
En Monte Providencia los formularios publicos viven en la cuenta de Marca
|
|
y todos los envios pasan por ahi, sin importar la sucursal destino. Por eso
|
|
este script ignora sucursales por defecto. Usa --location para forzar otra.
|
|
|
|
Paginacion:
|
|
GET /forms/submissions acepta page (default 1), limit (max 100) y
|
|
opcionalmente startAt / endAt (formato 'YYYY-MM-DD' o ISO 8601). El cliente
|
|
itera hasta agotar meta.nextPage, deduplicando por id.
|
|
|
|
Backfill historico:
|
|
SIN startAt/endAt, GHL solo devuelve ~1 mes hacia atras desde la fecha
|
|
actual. Para histórico completo hay que pedir ventanas mensuales. Este
|
|
script automatiza eso con --backfill-months N: parte la peticion en
|
|
ventanas mensuales (mes 1, mes 2, ...) y acumula todo en SQLite.
|
|
|
|
Uso:
|
|
python scripts/sync_forms_brand.py # sync incremental (mes corriente)
|
|
python scripts/sync_forms_brand.py --backfill-months 12 # ultimo año por ventanas mensuales
|
|
python scripts/sync_forms_brand.py --start-at 2025-09-01 # rango explicito desde fecha
|
|
python scripts/sync_forms_brand.py --start-at 2025-09-01 --end-at 2025-12-31
|
|
python scripts/sync_forms_brand.py --max 200 --quiet
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import re
|
|
import sys
|
|
import unicodedata
|
|
from datetime import date, datetime, timedelta
|
|
|
|
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
if ROOT_DIR not in sys.path:
|
|
sys.path.insert(0, ROOT_DIR)
|
|
|
|
import db # noqa: E402
|
|
import sync_engine # noqa: E402
|
|
|
|
BRAND_LOCATION_ID = "GbKkBpCmKu2QmloKFHy3"
|
|
|
|
# Field IDs historicos para el campo "Sucursal" en el formulario de Marca.
|
|
# Cuando se rediseña el form, GHL asigna un id nuevo al mismo campo conceptual.
|
|
# El primary se resuelve dinamicamente del schema del contacto; estos fallbacks
|
|
# cubren versiones viejas del form que ya no estan en el schema actual.
|
|
HISTORICAL_SUCURSAL_FIELD_IDS = {
|
|
BRAND_LOCATION_ID: ["ZVMoW8J0MqZNh3fEZ7Uu"],
|
|
}
|
|
|
|
|
|
def norm_field_name(name):
|
|
if not name:
|
|
return ""
|
|
t = unicodedata.normalize("NFKD", str(name).strip().lower())
|
|
t = "".join(ch for ch in t if not unicodedata.combining(ch))
|
|
return " ".join(t.split())
|
|
|
|
|
|
def resolve_sucursal_field_id(token, location_id):
|
|
"""Busca el field id del campo 'Sucursal' en el schema de contactos de la
|
|
location. Es el mismo id que aparece como key dentro de others en los
|
|
submissions, asi que vale para extraer el valor sin hardcodear."""
|
|
schema = sync_engine.ghl_client.get_object_schema(token, location_id, "contact")
|
|
for name, fid in schema.items():
|
|
if norm_field_name(name) == "sucursal":
|
|
return fid, name
|
|
return None, None
|
|
|
|
|
|
def parse_date_str(value):
|
|
"""Acepta YYYY-MM-DD o ISO 8601. Devuelve string normalizado para GHL."""
|
|
if not value:
|
|
return None
|
|
v = value.strip()
|
|
if re.match(r"^\d{4}-\d{2}-\d{2}$", v):
|
|
return v # GHL acepta YYYY-MM-DD directo
|
|
if re.match(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}", v):
|
|
return v
|
|
raise SystemExit(f"Fecha invalida {value!r}. Usa YYYY-MM-DD o ISO 8601.")
|
|
|
|
|
|
def first_of_month(d):
|
|
return d.replace(day=1)
|
|
|
|
|
|
def add_months(d, months):
|
|
"""Suma N meses a un date (sin dependencia externa)."""
|
|
y = d.year + (d.month - 1 + months) // 12
|
|
m = (d.month - 1 + months) % 12 + 1
|
|
# day=1 para evitar errores como feb 30
|
|
return d.replace(year=y, month=m, day=1)
|
|
|
|
|
|
def build_windows(start_at, end_at, backfill_months):
|
|
"""Devuelve lista de (start_str, end_str) en orden cronologico.
|
|
|
|
Reglas:
|
|
- Si hay --start-at y/o --end-at explicitos, se usa esa ventana (una sola).
|
|
- Si hay --backfill-months N, se generan N ventanas mensuales hasta hoy.
|
|
- Si no hay nada, devuelve [(None, None)] = comportamiento default (sin filtro).
|
|
"""
|
|
if start_at or end_at:
|
|
return [(start_at, end_at)]
|
|
if backfill_months and backfill_months > 0:
|
|
today = date.today()
|
|
# Empezamos backfill_months atras (al dia 1 de ese mes) y avanzamos
|
|
# mes a mes hasta cubrir hoy.
|
|
start = add_months(first_of_month(today), -backfill_months + 1)
|
|
windows = []
|
|
cursor = start
|
|
for _ in range(backfill_months + 1):
|
|
end = add_months(cursor, 1)
|
|
# ventana semicerrada [cursor, end)
|
|
windows.append((cursor.strftime("%Y-%m-%d"), end.strftime("%Y-%m-%d")))
|
|
cursor = end
|
|
if cursor > today:
|
|
break
|
|
return windows
|
|
return [(None, None)]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("--location", default=BRAND_LOCATION_ID,
|
|
help=f"Location de la que sincronizar formularios. Default: Marca ({BRAND_LOCATION_ID})")
|
|
parser.add_argument("--max", type=int, default=None,
|
|
help="Tope de submissions por formulario por ventana. Default: todos.")
|
|
parser.add_argument("--page-size", type=int, default=100,
|
|
help="Tamano de pagina (max 100). Default 100.")
|
|
parser.add_argument("--start-at", help="Fecha desde (YYYY-MM-DD o ISO). Filtra createdAt.")
|
|
parser.add_argument("--end-at", help="Fecha hasta (YYYY-MM-DD o ISO). Filtra createdAt.")
|
|
parser.add_argument("--backfill-months", type=int, default=0,
|
|
help="Backfill historico por N ventanas mensuales hacia atras. "
|
|
"Ignora si se pasa --start-at/--end-at.")
|
|
parser.add_argument("--reextract-only", action="store_true",
|
|
help="No descarga nada de GHL: solo re-procesa sucursal_value en "
|
|
"los submissions ya guardados usando los field_ids actual + historicos.")
|
|
parser.add_argument("--quiet", action="store_true", help="Reduce logs.")
|
|
args = parser.parse_args()
|
|
|
|
if hasattr(sys.stdout, "reconfigure"):
|
|
sys.stdout.reconfigure(encoding="utf-8")
|
|
|
|
if args.page_size < 1 or args.page_size > 100:
|
|
raise SystemExit("--page-size debe estar entre 1 y 100")
|
|
if args.backfill_months and args.backfill_months < 0:
|
|
raise SystemExit("--backfill-months debe ser >= 0")
|
|
|
|
start_at = parse_date_str(args.start_at)
|
|
end_at = parse_date_str(args.end_at)
|
|
windows = build_windows(start_at, end_at, args.backfill_months)
|
|
|
|
accounts = sync_engine.parse_accounts_csv()
|
|
account = next((a for a in accounts if a["location_id"] == args.location), None)
|
|
if not account:
|
|
raise SystemExit(f"Location {args.location} no esta en el CSV de tokens.")
|
|
|
|
token = account["token"]
|
|
location_id = account["location_id"]
|
|
print(f"=== SYNC FORMS: {account['nombre']} ({location_id}) ===")
|
|
print(f" Ventanas de fecha a consultar: {len(windows)}")
|
|
if len(windows) <= 3:
|
|
for s, e in windows:
|
|
print(f" [{s or '(sin filtro)'} -> {e or '(sin filtro)'}]")
|
|
|
|
sucursal_fid, sucursal_name = resolve_sucursal_field_id(token, location_id)
|
|
historical_fids = HISTORICAL_SUCURSAL_FIELD_IDS.get(location_id, [])
|
|
sucursal_field_ids = []
|
|
if sucursal_fid:
|
|
sucursal_field_ids.append(sucursal_fid)
|
|
print(f" Campo Sucursal actual: '{sucursal_name}' -> {sucursal_fid}")
|
|
else:
|
|
print(" WARN: no encontre el campo Sucursal actual en el schema de contactos.")
|
|
for hfid in historical_fids:
|
|
if hfid not in sucursal_field_ids:
|
|
sucursal_field_ids.append(hfid)
|
|
if historical_fids:
|
|
print(f" Field_ids historicos de Sucursal a tolerar: {historical_fids}")
|
|
|
|
# Modo re-extract: no hay sync nuevo, solo re-procesar registros existentes
|
|
if args.reextract_only:
|
|
if not sucursal_field_ids:
|
|
raise SystemExit("--reextract-only sin field_ids candidatos. Abortando.")
|
|
updated, total = db.reextract_sucursal_values(location_id, sucursal_field_ids)
|
|
print(f"\n=== RE-EXTRACT ===")
|
|
print(f" Submissions procesados: {total}")
|
|
print(f" Con sucursal poblada: {updated}")
|
|
print(f" Sin sucursal extraible: {total - updated}")
|
|
return
|
|
|
|
print("\n[1/2] Descargando catalogo de formularios...")
|
|
forms = sync_engine.ghl_client.get_forms(token, location_id)
|
|
print(f" Formularios: {len(forms)}")
|
|
for f in forms:
|
|
print(f" - {f.get('id')} {f.get('name')!r}")
|
|
db.save_forms(location_id, forms)
|
|
|
|
if not forms:
|
|
print("\nNo hay formularios. Nada que sincronizar.")
|
|
return
|
|
|
|
print("\n[2/2] Descargando submissions...")
|
|
total_synced = 0
|
|
seen_ids_global = set() # dedup cross-window
|
|
for form in forms:
|
|
form_id = form.get("id")
|
|
form_name = form.get("name") or form_id
|
|
|
|
for win_start, win_end in windows:
|
|
window_label = f"{win_start or 'open'}->{win_end or 'open'}"
|
|
|
|
def progress(page, page_count, accum, _form_name=form_name, _wl=window_label):
|
|
if args.quiet:
|
|
return
|
|
print(f" {_form_name} [{_wl}] page {page}: +{page_count} (acumulado {accum})")
|
|
|
|
subs = sync_engine.ghl_client.get_all_form_submissions(
|
|
token, location_id,
|
|
form_id=form_id,
|
|
page_size=args.page_size,
|
|
max_submissions=args.max,
|
|
start_at=win_start,
|
|
end_at=win_end,
|
|
progress_callback=progress,
|
|
)
|
|
# Dedup cross-window: el mismo submission puede aparecer en dos
|
|
# ventanas si las fechas se traslapan o si la API tiene jitter.
|
|
unique = [s for s in subs if s.get("id") and s["id"] not in seen_ids_global]
|
|
for s in unique:
|
|
seen_ids_global.add(s["id"])
|
|
|
|
if not unique:
|
|
if not args.quiet:
|
|
print(f" {form_name} [{window_label}]: 0 nuevos")
|
|
continue
|
|
|
|
n_saved = db.upsert_form_submissions(location_id, unique, sucursal_field_id=sucursal_field_ids)
|
|
total_synced += n_saved
|
|
print(f" {form_name} [{window_label}]: {n_saved} nuevos guardados (descargados {len(subs)}, unicos {len(unique)})")
|
|
|
|
print("\n" + "=" * 60)
|
|
db_total = db.count_form_submissions(location_id)
|
|
print(f"DONE. Submissions en SQLite para {location_id}: {db_total} (sync de hoy aporto {total_synced})")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|