(in-package #:lispsearch)

;;;; Indexing orchestration: Tier-1 catalog load + Tier-2 enrichment + refresh.

(defun load-catalog (db &key limit)
  "Fetch + parse the catalog and bulk-load names (Tier 1) in one transaction;
record catalog_sha and last_refresh. Returns the number of names loaded."
  (let* ((text (fetch-catalog))
         (sha (catalog-sha text))
         (pairs (parse-catalog text)))
    (when (and limit (< limit (length pairs)))
      (setf pairs (subseq pairs 0 limit)))
    (sqlite:with-transaction db
      (dolist (pair pairs)
        (upsert-name db (car pair) (cdr pair))))
    (meta-set db "catalog_sha" sha)
    (meta-set db "last_refresh" (iso-now))
    (length pairs)))

(defun pending-rows (db &key limit)
  "Rows still needing enrichment, as (id registry-name system-name)."
  (if limit
      (sqlite:execute-to-list db
        "SELECT id, registry_name, system_name FROM system
         WHERE enriched = 0 ORDER BY id LIMIT ?" limit)
      (sqlite:execute-to-list db
        "SELECT id, registry_name, system_name FROM system
         WHERE enriched = 0 ORDER BY id")))

(defun enrich-rows-serial (db rows &key (total (length rows)) progress)
  "Enrich ROWS one at a time on this thread. Returns the count enriched."
  (let ((done 0) (n 0))
    (dolist (row rows done)
      (incf n)
      (when (enrich-system db row) (incf done))
      (when (and progress (zerop (mod n progress)))
        (format *error-output* "~&; enriched ~D/~D~%" n total)))))

(defun enrich-rows-parallel (db rows &key (total (length rows)) (jobs *jobs*) progress)
  "Fetch enrichment for ROWS across JOBS lparallel workers, writing each result to
DB on this thread as it arrives (cl-sqlite writes must stay serialized). The
*registry* special and the output streams are propagated into the workers.
Returns the count enriched."
  (let ((lparallel:*kernel*
          (lparallel:make-kernel jobs :name "lispsearch-enrich"
                                 :bindings (list (cons '*standard-output* *standard-output*)
                                                 (cons '*error-output* *error-output*)
                                                 (cons '*registry* *registry*))))
        (done 0) (n 0))
    (unwind-protect
         (let ((channel (lparallel:make-channel)))
           (dolist (row rows)
             ;; pass ROW as a task arg (not a closure) to avoid the dolist
             ;; loop-variable capture pitfall.
             (lparallel:submit-task channel #'fetch-enrichment row))
           (dotimes (_ total done)
             (let ((data (lparallel:receive-result channel)))
               (incf n)
               (when data (upsert-system db data) (incf done))
               (when (and progress (zerop (mod n progress)))
                 (format *error-output* "~&; enriched ~D/~D~%" n total)))))
      (lparallel:end-kernel :wait t))))

(defun enrich-rows (db rows &key (jobs *jobs*) (progress nil))
  "Enrich ROWS into DB. Network fetches run in parallel across JOBS workers while
SQLite writes stay serialized on this thread. JOBS <= 1 (or a single row) runs
fully sequentially. Returns the count successfully enriched."
  (let* ((rows (coerce rows 'list))
         (total (length rows)))
    (if (or (null jobs) (<= jobs 1) (<= total 1))
        (enrich-rows-serial db rows :total total :progress progress)
        (enrich-rows-parallel db rows :total total
                                      :jobs (min jobs total) :progress progress))))

(defun build-index (db &key (enrich t) (jobs *jobs*) limit)
  "Cold build: load the catalog (Tier 1), then optionally enrich all pending
rows (Tier 2). Returns (values loaded-count enriched-count)."
  (let* ((loaded (load-catalog db :limit limit))
         (enriched (when enrich
                     (enrich-rows db (pending-rows db) :jobs jobs))))
    (values loaded (or enriched 0))))

(defun iso-days-ago (days)
  "ISO-8601 UTC timestamp DAYS in the past."
  (multiple-value-bind (sec min hr day mon yr)
      (decode-universal-time (- (get-universal-time) (* days 86400)) 0)
    (format nil "~4,'0D-~2,'0D-~2,'0DT~2,'0D:~2,'0D:~2,'0DZ" yr mon day hr min sec)))

(defun stale-rows (db cutoff-iso)
  "Rows that are un-enriched or whose fetched_at is older than CUTOFF-ISO."
  (sqlite:execute-to-list db
    "SELECT id, registry_name, system_name FROM system
     WHERE enriched = 0 OR fetched_at IS NULL OR fetched_at < ?
     ORDER BY id" cutoff-iso))

(defun refresh-index (db &key (max-age-days 30) (jobs *jobs*))
  "Re-fetch the catalog, add any new names, then re-enrich stale rows (un-enriched
or older than MAX-AGE-DAYS). Returns (values new-names re-enriched)."
  (let ((before (sqlite:execute-single db "SELECT COUNT(*) FROM system")))
    (load-catalog db)
    (let* ((after (sqlite:execute-single db "SELECT COUNT(*) FROM system"))
           (rows (stale-rows db (iso-days-ago max-age-days)))
           (re (enrich-rows db rows :jobs jobs)))
      (values (- after before) re))))
