;;; vulpea-db-extract.el --- Parse and extract note data -*- lexical-binding: t; -*-
;;
;; Copyright (c) 2015-2026 Boris Buliga <boris@d12frosted.io>
;;
;; Author: Boris Buliga <boris@d12frosted.io>
;; Maintainer: Boris Buliga <boris@d12frosted.io>
;;
;; This program is free software; you can redistribute it and/or
;; modify it under the terms of the GNU General Public License as
;; published by the Free Software Foundation, either version 3 of the
;; License, or (at your option) any later version.
;;
;; This program is distributed in the hope that it will be useful, but
;; WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;; General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program. If not, see
;; <http://www.gnu.org/licenses/>.
;;
;; Created: 16 Nov 2025
;;
;; URL: https://github.com/d12frosted/vulpea
;;
;; License: GPLv3
;;
;; This file is not part of GNU Emacs.
;;
;;; Commentary:
;;
;; Parser and extractor system for Vulpea v2.
;;
;; This module provides:
;; - Parse-once architecture with org-element
;; - Parse context shared across extractors
;; - Core extractors (notes, tags, links, meta)
;; - Extractor registry for plugins
;; - Configurable heading-level indexing
;;
;; Design:
;; - Single org-element parse per file
;; - Parse context contains AST + file metadata
;; - Extractors are pure functions: context -> data
;; - Registry allows plugins to add custom extractors
;;
;;; Code:

(require 'org-element)
(require 'org-id)
(require 'seq)
(require 'cl-lib)
(require 'vulpea-db)
(require 'vulpea-note)
(require 'vulpea-buffer)

(declare-function org-attach-dir "org-attach"
                  (&optional create-if-not-exists-p no-fs-check))
(declare-function org-attach-dir-from-id "org-attach"
                  (id &optional existing))

(defsubst vulpea-db--string-no-properties (value)
  "Return VALUE as a plain string without text properties."
  (when (and value (stringp value))
    (substring-no-properties value)))

(defun vulpea-db--strip-emphasis (str)
  "Strip Org emphasis markers from STR.

Removes emphasis markers for bold (*), italic (/), underline (_),
strikethrough (+), code (=), and verbatim (~).

Respects Org's emphasis rules: markers must be at word boundaries
and content must not start or end with whitespace.

Returns STR with emphasis markers removed, or nil if STR is nil."
  (when str
    (let ((result str))
      ;; Process each emphasis marker type
      ;; Pattern based on org-emph-re and org-verbatim-re structure:
      ;; - Pre: beginning of string OR space/punctuation
      ;; - Marker + non-whitespace content + same marker
      ;; - Post: end of string OR space/punctuation
      (dolist (marker '("\\*" "/" "_" "\\+" "=" "~"))
        (let ((re (concat
                   ;; Pre-marker context (group 1)
                   "\\(^\\|[-[:space:]('\"{]\\)"
                   ;; The emphasis: marker + content + marker (group 2 = full, group 3 = content)
                   marker
                   "\\([^[:space:]]\\|[^[:space:]][^" marker "]*[^[:space:]]\\)"
                   marker
                   ;; Post-marker context (group 4)
                   "\\([-[:space:].,:!?;'\")}\\[]\\|$\\)")))
          (while (string-match re result)
            (setq result (replace-match "\\1\\2\\3" nil nil result)))))
      result)))

(defun vulpea-db--extract-links-from-string (str &optional base-pos)
  "Extract org links from raw STR.

Returns list of plists with :dest, :type, :pos, and :description.
This is used for extracting links from keyword values like
#+TITLE or heading raw-values where org-element does not parse
links as elements.

BASE-POS is the buffer position of the start of STR.  Each
link's :pos is computed as BASE-POS + match offset within STR.
When BASE-POS is nil, the match offset within STR is used."
  (let ((result nil)
        (pos 0))
    (while (string-match org-link-bracket-re str pos)
      (let ((match-start (match-beginning 0))
            (description (match-string 2 str)))
        (setq pos (match-end 0))
        (let* ((raw-link (match-string 1 str))
               (type-and-path (if (string-match "\\`\\([^:]+\\):\\(.*\\)" raw-link)
                                  (cons (match-string 1 raw-link)
                                        (match-string 2 raw-link))
                                (cons "fuzzy" raw-link))))
          (push (list :dest (cdr type-and-path)
                      :type (car type-and-path)
                      :pos (+ (or base-pos 0) match-start)
                      :description description)
                result))))
    (nreverse result)))

(defsubst vulpea-db--strings-no-properties (values)
  "Return VALUES list with text properties stripped from each element."
  (when values
    (mapcar #'substring-no-properties values)))

(defun vulpea-db--extract-created-date (properties)
  "Extract date from CREATED property in PROPERTIES alist.

Supports various date formats:
- \"[2025-12-08 Sun 14:30]\" - Org timestamp with time
- \"[2025-12-08]\" - Org timestamp without time
- \"2025-12-08\" - ISO date

Returns date string in YYYY-MM-DD format, or nil if not found."
  (when-let* ((created (cdr (assoc "CREATED" properties))))
    (when (string-match "\\([0-9]\\{4\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)" created)
      (format "%s-%s-%s"
              (match-string 1 created)
              (match-string 2 created)
              (match-string 3 created)))))

;;; Archive Detection

(defun vulpea-db--archived-p (element properties filetags)
  "Check if ELEMENT is archived and should be excluded.

ELEMENT is either nil (for file-level) or a headline element.
PROPERTIES is the alist of properties for the element.
FILETAGS is the list of file-level tags.

Returns non-nil if the element is archived, which means:
- It has ARCHIVE_TIME property, or
- It has `org-archive-tag' directly or inherited from parent headlines,
  or from filetags."
  (when vulpea-db-exclude-archived
    (or
     ;; Check ARCHIVE_TIME property
     (assoc "ARCHIVE_TIME" properties)
     ;; Check for archive tag
     (let ((archive-tag (bound-and-true-p org-archive-tag)))
       (when archive-tag
         (or
          ;; Check filetags
          (member archive-tag filetags)
          ;; Check headline tags (direct and inherited)
          (when element
            (vulpea-db--headline-has-tag-p element archive-tag))))))))

(defun vulpea-db--headline-has-tag-p (headline tag)
  "Check if HEADLINE has TAG directly or inherited from ancestors."
  (or
   ;; Direct tag on this headline
   (member tag (org-element-property :tags headline))
   ;; Inherited from parent headline
   (let ((parent (org-element-property :parent headline)))
     (when (and parent (eq (org-element-type parent) 'headline))
       (vulpea-db--headline-has-tag-p parent tag)))))

;;; Parse Context

(cl-defstruct vulpea-parse-ctx
  "Context for parsing a single org file.

Slots:
  path          - Absolute file path
  ast           - Complete org-element AST
  file-node     - File-level heading data (plist)
  heading-nodes - List of heading-level data (plists)
  hash          - Content hash for change detection
  mtime         - File modification time
  size          - File size in bytes"
  path
  ast
  file-node
  heading-nodes
  hash
  mtime
  size)

;;; Extractor Definition

(cl-defstruct vulpea-extractor
  "Definition of a plugin extractor.

Slots:
  name         - Symbol identifying the extractor (required, unique)
  version      - Integer version number for migrations (default: 1)
  schema       - Database schema for plugin tables (optional)
  priority     - Execution priority, lower runs first (default: 100)
  extract-fn   - Function (ctx note-data) -> note-data (required).
                 Changes it makes to core note-data fields (:tags,
                 :links, :meta, ...) are persisted after all
                 extractors have run - both the materialized notes
                 row and the normalized tables are updated.  A nil
                 return means \"use note-data as it now stands\" -
                 in-place plist-put mutations included.
  worker-safe  - Whether extract-fn may run inside the extraction
                 worker in full-write mode (default: nil).  Requires
                 extract-fn to be a symbol the worker can resolve;
                 see worker-lib.  The extractor then writes its
                 tables through the worker's own database connection.
  worker-lib   - Feature symbol or absolute file path the worker
                 loads to make extract-fn available (optional; when
                 nil the function must already be defined after
                 loading vulpea).  If the worker cannot resolve the
                 function, it transparently falls back to streaming
                 results so the extractor runs in the main process -
                 nothing is lost, only the zero-freeze write.
  requires-ast - Whether extract-fn reads the parse context's AST.
                 Declare t when extract-fn calls
                 `vulpea-parse-ctx-ast': the file is then parsed at
                 the full object granularity regardless of
                 `vulpea-db-parse-granularity', and extraction never
                 runs in the async worker (the AST cannot cross the
                 process boundary).  Any other value means extract-fn
                 works purely from NOTE-DATA (an attachment scanner
                 reading :attach-dir, for example): extraction stays
                 fast and worker-eligible, and the extractor always
                 receives a context whose AST slot is nil - so an
                 undeclared AST reader visibly extracts nothing
                 instead of subtly missing inline objects.  The
                 default is the symbol `unset', which behaves like
                 nil but lets `vulpea-doctor' nudge authors to
                 declare their intent explicitly.

Example:
  (make-vulpea-extractor
   :name 'citations
   :version 1
   :priority 50
   :requires-ast t  ; my-extract-citations maps the AST
   :schema '((citations [(note-id :not-null)
                         (citekey :not-null)]
              (:foreign-key [note-id] :references notes [id]
               :on-delete :cascade)))
   :extract-fn #'my-extract-citations)"
  name
  (version 1)
  schema
  (priority 100)
  extract-fn
  (requires-ast 'unset)
  worker-safe
  worker-lib)

(defun vulpea-extractor-requires-ast-p (extractor)
  "Return non-nil when EXTRACTOR is a declared AST reader.
Only an explicit :requires-ast t counts; nil and the default `unset'
sentinel both mean the extractor works purely from note data and
receives a parse context whose AST slot is nil."
  (eq (vulpea-extractor-requires-ast extractor) t))

;;; Core Parsing

(defcustom vulpea-db-parse-method 'temp-buffer
  "Method to use for parsing org files during sync.

This setting has dramatic performance implications. Choose the mode
that matches your configuration:

  \\='single-temp-buffer (FASTEST)
    Reuses one hidden buffer and never re-runs `org-mode'.
    - ⚡ Best throughput (~1.3k files/sec)
    - ✗ Skips `org-mode-hook' entirely
    - ✗ Ignores per-file `#+TODO', `#+PROPERTY', `org-attach-dir'
    Use when your Org setup is 100% global.

  \\='temp-buffer (DEFAULT)
    Reuses one hidden buffer but re-runs `org-mode' per file.
    - ✓ Honors file-level keywords and `org-mode-hook'
    - ✓ Supports hook-based per-file tweaks
    - ⚠️ Slower if hooks are heavy (e.g., org-roam)
    Hooks can check `vulpea-db--active-parse-method' to skip work.

  \\='find-file (SLOWEST)
    Visits files with `find-file-noselect' as if opened manually.
    - ✓ Respects `.dir-locals.el' and file-visiting hooks
    - ✗ 30-40x slower than temp-buffer strategies

See README.org and bench/PERFORMANCE.md for guidance."
  :group 'vulpea
  :type '(choice (const :tag "Single temp buffer (fastest, skips hooks)" single-temp-buffer)
          (const :tag "Temp buffer (default, runs org-mode per file)" temp-buffer)
          (const :tag "Find file (slow, respects dir-locals)" find-file)))

(defcustom vulpea-db-parse-granularity 'element
  "Granularity of the org-element parse used during extraction.

  \\='element (DEFAULT)
    Parse down to elements only; links are located afterwards by
    scanning the textual elements with org's own link parser.
    2-3x faster on large files, since the bulk of a full parse is
    spent recognizing inline objects (bold, timestamps, entities...)
    that extraction never looks at.

  \\='object
    Full org-element parse including inline objects.  This is the
    historical behavior; keep it as an escape hatch if you observe
    any extraction difference and please report such cases.

Both modes are expected to produce identical extraction results;
the test suite locks their equivalence on an adversarial corpus.
Known difference: radio links (<<<target>>> matches in plain text)
are only picked up in \\='object mode.

Extractors registered via `vulpea-db-register-extractor' with an
explicit :requires-ast t declaration may inspect inline objects, so
their presence forces a full \\='object parse (see
`vulpea-db--effective-granularity').  Other extractors do not affect
this setting - they receive a parse context whose AST slot is nil."
  :group 'vulpea
  :type '(choice (const :tag "Element granularity (fast)" element)
          (const :tag "Object granularity (full parse)" object)))

(defcustom vulpea-db-index-plain-links t
  "Whether to index plain and angle links in note bodies.

When non-nil (default), every link org recognizes is indexed:
bracketed ([[id:...][desc]]), angle (<https://...>), and plain
\(https://... or a bare type:path in running text).

When nil, only bracketed links are indexed.  Locating unbracketed
links requires scanning all body text against `org-link-plain-re' -
a large alternation over every registered link type - which is by
far the most expensive part of link extraction (~600ms of a ~3.3s
10MB re-index).  Bracketed links are found with a cheap literal
search instead.  Since links that carry the note graph (id: links)
are practically always bracketed, collections that do not query
plain links can disable this for a noticeably shorter save freeze
on large files.

Applies to both parse granularities (see
`vulpea-db-parse-granularity'), so extraction output stays
identical between them either way.  Radio links count as
unbracketed.  Links in note titles are unaffected - titles only
ever carry bracketed links.

Changing this affects what lands in the database on the next
re-index of each file; run `vulpea-db-sync-full-scan' with FORCE to
apply it everywhere at once."
  :group 'vulpea
  :type 'boolean)

(defvar vulpea-db--extractors)

(defun vulpea-db--effective-granularity ()
  "Return the parse granularity to use for the current parse.

Honors `vulpea-db-parse-granularity', except when extractor plugins
declaring :requires-ast t are registered (they may map inline
objects like citations): those force \\='object granularity.
Extractors without that declaration do not - they receive a parse
context whose AST slot is nil."
  (if (seq-some #'vulpea-extractor-requires-ast-p vulpea-db--extractors)
      'object
    vulpea-db-parse-granularity))

(defvar vulpea-db--active-parse-method nil
  "Current `vulpea-db-parse-method' while parsing.

This is let-bound during parsing so hook authors can skip expensive
work by testing it with `bound-and-true-p'.")

(defvar vulpea-db--parse-buffer nil
  "Reusable buffer for parsing org files.
Caching the buffer with `org-mode' already initialized provides
significant performance improvement (12x faster) by avoiding
repeated `org-mode' activation overhead.")

(defvar vulpea-db--parse-buffer-run-hooks t
  "Whether `org-mode' should run hooks when initializing parse buffer.")

(defvar vulpea-db--timing-data nil
  "Accumulated timing data for profiling.
Format: ((phase . total-time-ms) ...)")

(defmacro vulpea-db--with-parse-buffer (&rest body)
  "Execute BODY in a reusable parse buffer with `org-mode' initialized.

The buffer is created once and reused across multiple file parses.
This avoids the expensive `org-mode' initialization overhead (0.7ms per file)
by initializing `org-mode' only once and reusing the buffer."
  (declare (indent 0))
  `(progn
     (unless (and vulpea-db--parse-buffer
              (buffer-live-p vulpea-db--parse-buffer))
      (setq vulpea-db--parse-buffer (generate-new-buffer " *vulpea-parse*"))
      (with-current-buffer vulpea-db--parse-buffer
       (if vulpea-db--parse-buffer-run-hooks
           (org-mode)
         (delay-mode-hooks (org-mode)))))
     (with-current-buffer vulpea-db--parse-buffer
      ,@body)))

(defun vulpea-db--parse-with-temp-buffer (path rerun-org-mode)
  "Parse org file at PATH using shared temp buffer.

If RERUN-ORG-MODE is non-nil, `org-mode' (and its hooks) are executed
after loading PATH so file-local keywords and hooks are respected."
  (let ((vulpea-db--parse-buffer-run-hooks rerun-org-mode))
    (vulpea-db--with-parse-buffer
      (unwind-protect
          (let* ((t0 (current-time))

                 ;; Required in case of relative `org-attach-dir'
                 (_ (setq buffer-file-name path
                          default-directory (file-name-directory path)))

                 (_ (let ((inhibit-read-only t)
                          (inhibit-modification-hooks t))
                      (erase-buffer)
                      (insert-file-contents path)))

                 ;; Reset org-element cache after replacing buffer
                 ;; contents with inhibit-modification-hooks.  Without
                 ;; this, the cache retains stale positions from the
                 ;; previous file and org-element-parse-buffer hits
                 ;; "Invalid search bound" or marker errors.
                 (_ (when (fboundp 'org-element-cache-reset)
                      (org-element-cache-reset)))

                 (t1 (current-time))
                 (_ (when rerun-org-mode
                      (let ((delay-mode-hooks nil))
                        (org-mode))))
                 (t2 (current-time))
                 (ast (org-element-parse-buffer (vulpea-db--effective-granularity)))
                 (t3 (current-time))
                 (attrs (file-attributes path))
                 (mtime (float-time (file-attribute-modification-time attrs)))
                 (size (file-attribute-size attrs))
                 ;; Hash the buffer directly - copying a large buffer
                 ;; to a string first costs time and garbage
                 (hash (secure-hash 'sha256 (current-buffer)))
                 (file-title (vulpea-db--extract-file-title ast path))
                 (file-node (vulpea-db--extract-file-node ast path (current-buffer) file-title))
                 (heading-nodes (vulpea-db--extract-heading-nodes ast path (current-buffer) file-title))
                 (t4 (current-time)))

            ;; Accumulate detailed timing if enabled
            (when vulpea-db--timing-data
              (let ((io-time (* 1000 (float-time (time-subtract t1 t0))))
                    (org-mode-time (* 1000 (float-time (time-subtract t2 t1))))
                    (parse-ast-time (* 1000 (float-time (time-subtract t3 t2))))
                    (extract-time (* 1000 (float-time (time-subtract t4 t3)))))
                (dolist (entry `((parse-io . ,io-time)
                                 (parse-org-mode . ,org-mode-time)
                                 (parse-ast . ,parse-ast-time)
                                 (parse-extract . ,extract-time)))
                  (let ((existing (assoc (car entry) vulpea-db--timing-data)))
                    (if existing
                        (setcdr existing (+ (cdr existing) (cdr entry)))
                      (push entry vulpea-db--timing-data))))))

            (make-vulpea-parse-ctx
             :path path
             :ast ast
             :file-node file-node
             :heading-nodes heading-nodes
             :hash hash
             :mtime mtime
             :size size))
        ;; Always clear buffer state - even when parsing above errors -
        ;; so the reused parse buffer is not left associated with PATH
        ;; (which would mark it modified and risk file-change tracking).
        (setq buffer-file-name nil)
        (set-buffer-modified-p nil)))))

(defun vulpea-db--parse-file (path)
  "Parse org file at PATH and return parse context.

Returns `vulpea-parse-ctx' structure with:
- Full org-element AST
- File-level node data
- Heading-level node data (if enabled)
- File metadata (hash, mtime, size)

Respects `vulpea-db-parse-method' setting for parsing approach.

For non-.org files (e.g., .org.age, .org.gpg), always uses the
`find-file' method regardless of `vulpea-db-parse-method' to
ensure decryption hooks run properly."
  (let* ((vulpea-db--active-parse-method vulpea-db-parse-method)
         (method (if (string-suffix-p ".org" path)
                     vulpea-db-parse-method
                   'find-file)))
    (pcase method
      ('single-temp-buffer
       (vulpea-db--parse-with-temp-buffer path nil))

      ('temp-buffer
       (vulpea-db--parse-with-temp-buffer path t))

      ('find-file
       ;; Use find-file-noselect: slower but respects hooks and dir-locals
       (let ((buffer (find-file-noselect path t)))
         (unwind-protect
             (with-current-buffer buffer
               (let* ((ast (org-element-parse-buffer (vulpea-db--effective-granularity)))
                      (attrs (file-attributes path))
                      (mtime (float-time (file-attribute-modification-time attrs)))
                      (size (file-attribute-size attrs))
                      (hash (secure-hash 'sha256 (current-buffer)))
                      (file-title (vulpea-db--extract-file-title ast path)))

                 (make-vulpea-parse-ctx
                  :path path
                  :ast ast
                  :file-node (vulpea-db--extract-file-node ast path (current-buffer) file-title)
                  :heading-nodes (vulpea-db--extract-heading-nodes ast path (current-buffer) file-title)
                  :hash hash
                  :mtime mtime
                  :size size)))
           ;; Always kill the buffer after parsing
           (when (buffer-live-p buffer)
             (kill-buffer buffer)))))

      (_
       (error "Unsupported vulpea-db-parse-method: %s" vulpea-db-parse-method)))))

(defun vulpea-db--attach-dir-props-p (buffer)
  "Return non-nil when BUFFER may set attach dirs through properties.

Scans for DIR / ATTACH_DIR in property drawers or #+PROPERTY
keywords, and consults `org-keyword-properties' - a #+SETUPFILE can
deliver a DIR property that never appears in the buffer text but
that `org-attach-dir' would honor through inheritance.  When none
are present, `org-attach-dir' can only ever derive the attachment
directory from the node's own ID, so extraction may skip the
per-node property lookups entirely (see `vulpea-db--attach-dir')."
  (with-current-buffer buffer
    (or (assoc-string "DIR" (bound-and-true-p org-keyword-properties) t)
        (assoc-string "ATTACH_DIR" (bound-and-true-p org-keyword-properties) t)
        (save-excursion
          (goto-char (point-min))
          (let ((case-fold-search t))
            (re-search-forward
             "^[ \t]*\\(?::\\(?:DIR\\|ATTACH_DIR\\)\\+?:\\|#\\+PROPERTY:[ \t]*\\(?:DIR\\|ATTACH_DIR\\)\\)"
             nil t))))))

(defun vulpea-db--attach-dir (buffer pos id props-p)
  "Compute attachment directory for the note with ID at POS in BUFFER.

When PROPS-P is nil the buffer contains no DIR/ATTACH_DIR properties
\(per `vulpea-db--attach-dir-props-p'), so the result is derived from
ID directly - the same value `org-attach-dir' would return, minus its
three per-node `org-entry-get' tree walks, which dominate heading
extraction cost on large files (issue #359).  Otherwise falls back to
`org-attach-dir' at POS."
  (with-current-buffer buffer
    (require 'org-attach)
    (if props-p
        (save-excursion
          (goto-char pos)
          (org-attach-dir nil 'no-fs-check))
      (org-attach-dir-from-id id 'existing))))

(defun vulpea-db--extract-file-title (ast path)
  "Extract file title from AST at PATH.

Returns the #+TITLE keyword value if present, otherwise the filename base."
  (let ((keywords (org-element-map ast 'keyword
                    (lambda (kw)
                      (cons (vulpea-db--string-no-properties
                             (org-element-property :key kw))
                            (vulpea-db--string-no-properties
                             (org-element-property :value kw)))))))
    (or (when-let* ((title (cdr (assoc "TITLE" keywords))))
          (vulpea-db--strip-emphasis
           (org-link-display-format title)))
        (file-name-base path))))

(defun vulpea-db--extract-file-node (ast _path buffer file-title)
  "Extract file-level node data from AST in BUFFER.

_PATH is accepted for signature symmetry with
`vulpea-db--extract-heading-nodes' but is not used.
FILE-TITLE is the title of the file (from #+TITLE or filename).

Returns plist with:
  :id :title :aliases :tags :links :properties :meta
  :todo :priority :scheduled :deadline :closed :attach-dir :file-title

Returns nil if:
- File has no ID property in property drawer
- File has `vulpea-db-exclude-property' set to non-nil value.
- File is archived (when `vulpea-db-exclude-archived' is non-nil)"
  (let* ((keywords (org-element-map ast 'keyword
                     (lambda (kw)
                       (cons (vulpea-db--string-no-properties
                              (org-element-property :key kw))
                             (vulpea-db--string-no-properties
                              (org-element-property :value kw))))))
         (properties (vulpea-db--extract-properties ast nil))
         (category-kw (cdr (assoc "CATEGORY" keywords)))
         (properties (if (and category-kw
                              (not (assoc "CATEGORY" properties)))
                         (append properties
                                 (list (cons "CATEGORY" category-kw)))
                       properties))
         (id (cdr (assoc "ID" properties)))
         (ignored (org-not-nil (cdr (assoc vulpea-db-exclude-property properties))))
         (filetags (cl-mapcan (lambda (kw)
                                (when (string= "FILETAGS" (car kw))
                                  (split-string (cdr kw) ":" t)))
                              keywords))
         (archived (vulpea-db--archived-p nil properties filetags)))

    ;; Only index if ID exists, not explicitly ignored, and not archived
    (when (and id (not ignored) (not archived))
      (let* ((title-kw (org-element-map ast 'keyword
                         (lambda (kw)
                           (when (string= "TITLE"
                                          (org-element-property :key kw))
                             kw))
                         nil t))
             (raw-title (when title-kw
                          (vulpea-db--string-no-properties
                           (org-element-property :value title-kw))))
             (title-value-pos
              (when title-kw
                (+ (org-element-property :begin title-kw)
                   ;; skip "#+TITLE: " prefix to get to value
                   (length "#+TITLE: "))))
             (title-links (when raw-title
                            (vulpea-db--extract-links-from-string
                             raw-title title-value-pos)))
             (meta (vulpea-db--extract-meta ast))
             (links (append title-links
                            (vulpea-db--extract-links ast t)))  ; Don't recurse into headlines
             (aliases (vulpea-db--extract-aliases properties))
             (attach-dir (vulpea-db--attach-dir
                          buffer (point-min) id
                          (vulpea-db--attach-dir-props-p buffer))))
        (list :id id
              :title file-title
              :aliases aliases
              :tags filetags
              :links links
              :properties properties
              :meta meta
              :todo nil
              :priority nil
              :scheduled nil
              :deadline nil
              :closed nil
              :attach-dir attach-dir
              :file-title file-title)))))

(defun vulpea-db--extract-closed-from-logbook (section buffer)
  "Extract the most recent completion timestamp from SECTION's LOGBOOK.

SECTION is the `section' element of a headline (its own direct
content, excluding any child headlines).  BUFFER is the buffer the
AST was parsed from, used to read the raw drawer text.

Scans the LOGBOOK drawer for state-change log entries of the form

    - State \"DONE\" from \"...\" [timestamp]

and returns the raw timestamp string of the most recent such entry
\(for example \"[2024-08-13 Tue 09:41]\"), or nil when SECTION has no
LOGBOOK drawer or no matching entry.

This is a fallback for the common `org-mode' workflow where the
completion time is recorded only in the LOGBOOK (via `org-log-done'
state logging) rather than on a `CLOSED:' planning line."
  (when (and section buffer)
    (let (timestamps)
      (org-element-map section 'drawer
        (lambda (drawer)
          (when (string= "LOGBOOK"
                         (upcase (or (org-element-property :drawer-name drawer)
                                     "")))
            (let ((beg (org-element-property :contents-begin drawer))
                  (end (org-element-property :contents-end drawer)))
              (when (and beg end)
                (with-current-buffer buffer
                  (save-excursion
                    (save-match-data
                      (goto-char beg)
                      (while (re-search-forward
                              "State[ \t]+\"DONE\"[^][\n]*\\(\\[[^]\n]+\\]\\)"
                              end t)
                        (push (match-string-no-properties 1)
                              timestamps))))))))))
      ;; Log timestamps share a fixed-width \"[YYYY-MM-DD ...\" prefix, so
      ;; a lexicographic comparison orders them chronologically.  The
      ;; greatest is the most recent completion.
      (car (sort timestamps #'string>)))))

(defun vulpea-db--extract-heading-nodes (ast path buffer file-title)
  "Extract heading-level nodes from AST at PATH in BUFFER.

FILE-TITLE is the title of the file containing the headings.

Returns list of plists, one per heading with ID property.
Each plist has same structure as file-node.

Skips headings that:
- Have no ID property
- Have `vulpea-db-exclude-property' set to non-nil value.
- Are archived (when `vulpea-db-exclude-archived' is non-nil)

Respects `vulpea-db-index-heading-level' setting."
  (when (vulpea-db--should-index-headings-p path)
    ;; Extract filetags once for archive checking and tag inheritance
    (let* ((filetags (apply #'append
                            (org-element-map ast 'keyword
                              (lambda (kw)
                                (when (string= "FILETAGS"
                                               (org-element-property :key kw))
                                  (split-string
                                   (org-element-property :value kw)
                                   ":" t))))))
           ;; One scan for DIR/ATTACH_DIR properties gates the cheap
           ;; ID-derived attach-dir path for every heading
           (attach-props-p (vulpea-db--attach-dir-props-p buffer)))
      (org-element-map ast 'headline
        (lambda (headline)
          (when-let* ((id (org-element-property :ID headline)))
            (let* ((properties (vulpea-db--extract-properties ast headline))
                   (ignored (org-not-nil (cdr (assoc vulpea-db-exclude-property properties))))
                   (archived (vulpea-db--archived-p headline properties filetags)))
              ;; Only index if not explicitly ignored and not archived
              (unless (or ignored archived)
                (let* ((title (vulpea-db--strip-emphasis
                               (org-link-display-format
                                (vulpea-db--string-no-properties
                                 (org-element-property :raw-value headline)))))
                       (inherited-tags
                        (when org-use-tag-inheritance
                          (let (parent-tags
                                (current headline))
                            ;; Walk bottom-up, but build top-down order
                            ;; by prepending each level's tags to the
                            ;; accumulator (outermost ends up first).
                            (while (setq current (org-element-property :parent current))
                              (when (eq (org-element-type current) 'headline)
                                (let (level-tags)
                                  (dolist (tag (vulpea-db--strings-no-properties
                                               (org-element-property :tags current)))
                                    (when (org-tag-inherit-p tag)
                                      (push tag level-tags)))
                                  (setq parent-tags
                                        (append (nreverse level-tags)
                                                parent-tags)))))
                            ;; Prepend inheritable filetags
                            (let (filtered-filetags)
                              (dolist (tag filetags)
                                (when (org-tag-inherit-p tag)
                                  (push tag filtered-filetags)))
                              (append (nreverse filtered-filetags)
                                      parent-tags)))))
                       (tags (seq-uniq
                              (append inherited-tags
                                      (vulpea-db--strings-no-properties
                                       (org-element-property :tags headline)))))
                       (level (org-element-property :level headline))
                       (pos (org-element-property :begin headline))
                       (todo (vulpea-db--string-no-properties
                              (org-element-property :todo-keyword headline)))
                       (priority (org-element-property :priority headline))
                       (scheduled (org-element-property :scheduled headline))
                       (deadline (org-element-property :deadline headline))
                       (closed (org-element-property :closed headline))
                       ;; Extract meta from the section element (first child of headline)
                       ;; because org-element-map with NO-RECURSION 'headline won't
                       ;; descend into a headline element itself
                       (section (seq-find (lambda (el)
                                            (eq (org-element-type el) 'section))
                                          (org-element-contents headline)))
                       (meta (when section
                               (vulpea-db--extract-meta section)))
                       (closed-str
                        (or (when closed
                              (vulpea-db--string-no-properties
                               (org-element-property :raw-value closed)))
                            ;; Fallback: tasks whose completion time is
                            ;; recorded only in the LOGBOOK drawer (State
                            ;; "DONE" entries) rather than on a CLOSED
                            ;; planning line.  Gate on done-state to mirror
                            ;; org dropping CLOSED when a task is reopened.
                            (when (eq (org-element-property :todo-type headline)
                                      'done)
                              (vulpea-db--extract-closed-from-logbook
                               section buffer))))
                       (raw-title (vulpea-db--string-no-properties
                                   (org-element-property :raw-value headline)))
                       (title-start (+ pos level 1 ; stars + space
                                     (if todo (+ (length todo) 1) 0)
                                     (if priority 5 0))) ; "[#X] " incl. trailing space
                       (title-links (vulpea-db--extract-links-from-string
                                     raw-title title-start))
                       (links (append title-links
                                      (vulpea-db--extract-links headline t)))
                       (outline-path (let (path
                                           (current headline))
                                       (while (setq current (org-element-property :parent current))
                                         (when (eq (org-element-type current) 'headline)
                                           (push (vulpea-db--strip-emphasis
                                                  (org-link-display-format
                                                   (vulpea-db--string-no-properties
                                                    (org-element-property :raw-value current))))
                                                 path)))
                                       path))
                       (attach-dir (vulpea-db--attach-dir
                                    buffer pos id attach-props-p)))

                  (list :id id
                        :level level
                        :pos pos
                        :title title
                        :aliases (vulpea-db--extract-aliases properties)
                        :tags tags
                        :links links
                        :properties properties
                        :meta meta
                        :todo todo
                        :priority priority
                        :scheduled (when scheduled
                                     (vulpea-db--string-no-properties
                                      (org-element-property :raw-value scheduled)))
                        :deadline (when deadline
                                    (vulpea-db--string-no-properties
                                     (org-element-property :raw-value deadline)))
                        :closed closed-str
                        :outline-path outline-path
                        :attach-dir attach-dir
                        :file-title file-title))))))))))

(defun vulpea-db--should-index-headings-p (path)
  "Check if headings should be indexed for PATH.

Respects `vulpea-db-index-heading-level' setting:
- t: always index headings
- nil: never index headings
- function: call with path, use result"
  (pcase vulpea-db-index-heading-level
    ('t t)
    ('nil nil)
    ((pred functionp) (funcall vulpea-db-index-heading-level path))
    (_ t)))

;;; Extractors

(defun vulpea-db--extract-aliases (properties)
  "Extract aliases from PROPERTIES alist.

Looks for property defined by `vulpea-buffer-alias-property'.
Handles both quoted aliases (with spaces) and unquoted aliases properly.
Strips org links and emphasis markers from each alias.

The property name is upcased before lookup because PROPERTIES keys
are stored upcased, so a lowercase or mixed-case
`vulpea-buffer-alias-property' still matches."
  (when-let* ((aliases-str (cdr (assoc (upcase vulpea-buffer-alias-property)
                                       properties))))
    (setq aliases-str (string-trim aliases-str))
    (let ((result nil)
          (pos 0))
      (while (< pos (length aliases-str))
        ;; Skip whitespace
        (while (and (< pos (length aliases-str))
                    (= (aref aliases-str pos) ? ))
          (setq pos (1+ pos)))
        (when (< pos (length aliases-str))
          (let ((char (aref aliases-str pos)))
            (cond
             ;; Quoted alias - find matching closing quote
             ((= char ?\")
              (let ((end (string-match "\"" aliases-str (1+ pos))))
                (if end
                    (progn
                      (push (vulpea-db--strip-emphasis
                             (org-link-display-format
                              (substring aliases-str (1+ pos) end)))
                            result)
                      (setq pos (1+ end)))
                  (error "Unmatched quote in %s" vulpea-buffer-alias-property))))
             ;; Unquoted alias - find next space or end of string
             (t
              (let ((end (or (string-match " " aliases-str pos)
                             (length aliases-str))))
                (push (vulpea-db--strip-emphasis
                       (org-link-display-format
                        (substring aliases-str pos end)))
                      result)
                (setq pos end)))))))
      (nreverse result))))

(defun vulpea-db--extract-properties (ast-or-node &optional headline)
  "Extract properties from AST-OR-NODE.

If HEADLINE is provided, extract from that headline.
Otherwise extract from file-level (property drawer).

Returns alist of (key . value) pairs."
  (let ((node (or headline ast-or-node)))
    (org-element-map node 'property-drawer
      (lambda (drawer)
        (org-element-map drawer 'node-property
          (lambda (prop)
            (cons (upcase (vulpea-db--string-no-properties
                          (org-element-property :key prop)))
                  (vulpea-db--string-no-properties
                   (org-element-property :value prop))))))
      nil t
      ;; When extracting file-level properties, don't recurse into
      ;; headlines — otherwise the first heading's property drawer
      ;; is mistakenly returned as the file-level one.
      (unless headline 'headline))))

(defun vulpea-db--extract-links (ast-or-node &optional no-recursion)
  "Extract all links from AST-OR-NODE.

Returns list of plists with :dest, :type, and :pos.
Captures all link types: id:, roam:, file:, http:, https:,
attachment:, elisp:, and any other `org-mode' link type.

If NO-RECURSION is non-nil, stops recursion at note boundaries
\(headlines with an ID property).  Links inside non-note subtrees
are collected as part of the current node.  This prevents links
from child notes leaking to the parent while still capturing
links from plain (non-note) subtrees."
  (if no-recursion
      (vulpea-db--extract-links-stopping-at-notes ast-or-node)
    ;; Normal case: search everything without restrictions
    (org-element-map ast-or-node 'link
      (lambda (link)
        (let ((type (org-element-property :type link))
              (path (org-element-property :path link))
              (pos (org-element-property :begin link))
              (desc (when-let* ((contents (org-element-contents link)))
                      (substring-no-properties
                       (org-element-interpret-data contents)))))
          (when (and type path
                     (or vulpea-db-index-plain-links
                         (eq (org-element-property :format link) 'bracket)))
            (list :dest path :type type :pos pos :description desc)))))))

(defun vulpea-db--extract-links-stopping-at-notes (node)
  "Extract links from NODE, stopping at note boundaries.

Collects links from NODE but does not descend into child
headlines that have an ID property (note boundaries).  Links
inside non-note child headlines are included."
  (let ((result nil))
    (vulpea-db--walk-links-skipping-notes node (lambda (link) (push link result)))
    (nreverse result)))

(defun vulpea-db--region-links (start end callback)
  "Collect links between START and END in the current buffer.

Used for element-granularity ASTs, where textual elements carry no
parsed objects.  Candidates are located with `org-link-any-re' - or,
when `vulpea-db-index-plain-links' is nil, with a much cheaper
literal search for \"[[\" - and parsed with
`org-element-link-parser', org's own parser, so type, path, and
description semantics match a full object parse.

CALLBACK is called with a plist (:dest :type :pos :description) for
each link found.

Links inside inline verbatim/code markup (=...= and ~...~) or macro
calls ({{{...}}}) are excluded, matching the full object parse:
those objects hide their contents from org's link recognition."
  (let ((exclusions (vulpea-db--region-link-exclusions start end)))
    (save-excursion
      (goto-char start)
      (while (if vulpea-db-index-plain-links
                 (re-search-forward org-link-any-re end t)
               (search-forward "[[" end t))
        ;; Capture the match bound before calling the parser:
        ;; `org-element-link-parser' runs its own regexps and clobbers
        ;; the global match data, so reading `match-end' after it can
        ;; move point backwards and loop forever
        (let ((candidate-end (match-end 0)))
          (if (vulpea-db--pos-excluded-p (match-beginning 0) exclusions)
              (goto-char candidate-end)
            (goto-char (match-beginning 0))
            (let ((link (org-element-link-parser)))
              (if (not link)
                  (goto-char candidate-end)
                (let ((type (org-element-property :type link))
                      (path (org-element-property :path link))
                      (pos (org-element-property :begin link))
                      (cb (org-element-property :contents-begin link))
                      (ce (org-element-property :contents-end link)))
                  (when (and type path)
                    (funcall callback
                             (list :dest path :type type :pos pos
                                   :description (when (and cb ce)
                                                  (buffer-substring-no-properties
                                                   cb ce)))))
                  ;; Jumping to the link's end also skips over any plain
                  ;; link inside this link's description, which a full
                  ;; object parse would not surface either.  The link end
                  ;; is strictly after the match beginning, so the loop
                  ;; always advances.
                  (goto-char (min end (org-element-property :end link))))))))))))

(defun vulpea-db--region-link-exclusions (start end)
  "Return spans in [START, END) whose contents hide links from org.
Covers inline verbatim/code markup (`org-verbatim-re') and macro
calls; a full object parse never surfaces links inside either."
  (let (spans)
    (save-excursion
      (goto-char start)
      (while (re-search-forward org-verbatim-re end t)
        (push (cons (match-beginning 2) (match-end 2)) spans)
        ;; Overlapping emphasis: resume inside the match so adjacent
        ;; spans are still found, but always advance
        (goto-char (max (1+ (match-beginning 2)) (match-end 2))))
      (goto-char start)
      (while (re-search-forward "{{{[^}\n]*}}}" end t)
        (push (cons (match-beginning 0) (match-end 0)) spans)))
    spans))

(defun vulpea-db--pos-excluded-p (pos exclusions)
  "Return non-nil when POS falls inside one of EXCLUSIONS."
  (seq-some (lambda (span)
              (and (>= pos (car span)) (< pos (cdr span))))
            exclusions))

(defun vulpea-db--walk-links-skipping-notes (node callback)
  "Walk NODE collecting links via CALLBACK, skipping note headlines.

CALLBACK is called with a plist (:dest :type :pos :description) for
each link.  Descends into child headlines only if they lack an ID
property.

Works on both object- and element-granularity ASTs: parsed link
objects are collected directly, while textual elements that carry no
parsed objects (element granularity) are scanned in the buffer via
`vulpea-db--region-links'.  For the latter the current buffer must be
the buffer NODE was parsed from."
  (dolist (child (org-element-contents node))
    (let ((child-type (org-element-type child)))
      (cond
       ;; Link element: collect it
       ((eq child-type 'link)
        (let ((type (org-element-property :type child))
              (path (org-element-property :path child))
              (pos (org-element-property :begin child))
              (desc (when-let* ((contents (org-element-contents child)))
                      (substring-no-properties
                       (org-element-interpret-data contents)))))
          (when (and type path
                     (or vulpea-db-index-plain-links
                         (eq (org-element-property :format child) 'bracket)))
            (funcall callback (list :dest path :type type :pos pos
                                    :description desc)))))
       ;; Headline with ID: skip (note boundary)
       ((and (eq child-type 'headline)
             (org-element-property :ID child))
        nil)
       ;; Headline without ID: extract title links, then recurse
       ((eq child-type 'headline)
        (let* ((raw-value (org-element-property :raw-value child))
               (h-level (org-element-property :level child))
               (h-todo (org-element-property :todo-keyword child))
               (h-priority (org-element-property :priority child))
               (h-begin (org-element-property :begin child))
               (title-start (+ h-begin h-level 1
                               (if h-todo (+ (length h-todo) 1) 0)
                               (if h-priority 4 0))))
          (when raw-value
            (dolist (link (vulpea-db--extract-links-from-string
                          raw-value title-start))
              (funcall callback link))))
        (vulpea-db--walk-links-skipping-notes child callback))
       ;; Textual element without parsed objects (element
       ;; granularity): scan its buffer region for links
       ((and (memq child-type '(paragraph verse-block table-row))
             (not (org-element-contents child))
             (org-element-property :contents-begin child))
        (vulpea-db--region-links
         (org-element-property :contents-begin child)
         (org-element-property :contents-end child)
         callback))
       ;; Anything else (section, paragraph, etc.): recurse
       ((org-element-contents child)
        (vulpea-db--walk-links-skipping-notes child callback))))))

(defun vulpea-db--normalize-timestamps (string)
  "Normalize org timestamps in STRING as an object parse prints them.
Day names are canonicalized (sat becomes Sat); anything
`org-timestamp-from-string' cannot parse is left untouched."
  (if (not (string-match-p org-ts-regexp-both string))
      string
    (replace-regexp-in-string
     org-ts-regexp-both
     (lambda (match)
       ;; The parser runs its own regexps; protect the match data
       ;; replace-regexp-in-string is iterating with
       (save-match-data
         (if-let* ((ts (org-timestamp-from-string match)))
             (org-element-interpret-data ts)
           match)))
     string t t)))

(defun vulpea-db--extract-meta (element)
  "Extract metadata from ELEMENT (AST or headline element).

Metadata is defined by the first description list:
  - key :: value
  - key :: value2

Returns alist of (key . values) where values is list of strings.
Link values are stored as interpreted strings.

Works on both object- and element-granularity ASTs.  At element
granularity the item tag is a raw string and the value element
carries no parsed objects, so the value is read from the buffer
directly; the current buffer must then be the buffer ELEMENT was
parsed from."
  (let* ((pls (org-element-map element 'plain-list #'identity nil nil 'headline))
         (pl (seq-find
              (lambda (pl)
                (equal 'descriptive
                       (org-element-property :type pl)))
              pls))
         (meta-alist nil))
    (when pl
      (let ((items (org-element-map pl 'item #'identity)))
        (dolist (item items)
          (let* ((tag-contents (org-element-property :tag item))
                 (key (cond
                       ;; Element granularity: tag is a raw string
                       ((stringp tag-contents)
                        (substring-no-properties (string-trim tag-contents)))
                       (tag-contents
                        (substring-no-properties
                         (string-trim
                          (org-element-interpret-data
                           (org-element-contents tag-contents)))))))
                 (value-el (car (org-element-contents item)))
                 (value (cond
                         ((and value-el (org-element-contents value-el))
                          (substring-no-properties
                           (string-trim
                            (org-element-interpret-data value-el))))
                         ;; Element granularity: no parsed objects,
                         ;; read the raw buffer text.  Timestamps are
                         ;; re-interpreted so day names normalize the
                         ;; way a full object parse would print them.
                         (value-el
                          (when-let* ((cb (org-element-property
                                           :contents-begin value-el))
                                      (ce (org-element-property
                                           :contents-end value-el)))
                            (vulpea-db--normalize-timestamps
                             (string-trim
                              (buffer-substring-no-properties cb ce))))))))
            (when (and key value)
              (let ((existing (assoc key meta-alist)))
                (if existing
                    (setcdr existing (append (cdr existing) (list value)))
                  (push (cons key (list value)) meta-alist))))))))
    meta-alist))

;;; Extractor Registry

(defvar vulpea-db--extractors nil
  "List of registered extractors.

Each extractor is a `vulpea-extractor' struct.")

(defun vulpea-db--apply-plugin-schema (extractor)
  "Apply database schema from EXTRACTOR if present.

Creates tables and indices defined in the extractor's :schema field.
Schema format matches `vulpea-db--schema':
  ((table-name
    [(column-name :constraints...)]
    (:unique [columns])
    (:foreign-key [columns] :references table [columns]))
   ...)

Also registers the schema version in schema-registry table."
  (when-let* ((schema (vulpea-extractor-schema extractor))
             (name (vulpea-extractor-name extractor))
             (version (vulpea-extractor-version extractor)))
    (let ((db (vulpea-db)))
      ;; Check if schema already applied at this version
      (let ((existing-version
             (caar (emacsql db
                            [:select [version] :from schema-registry
                             :where (= name $s1)]
                            (symbol-name name)))))
        (unless (and existing-version (>= existing-version version))
          ;; Create tables from schema
          (dolist (table-spec schema)
            (emacsql db [:create-table :if-not-exists $i1 $S2]
                     (car table-spec)
                     (cdr table-spec)))

          ;; Register schema version
          (emacsql db [:insert :or :replace :into schema-registry
                       :values $v1]
                   (list (vector (symbol-name name)
                                 version
                                 (format-time-string "%Y-%m-%d %H:%M:%S")))))))))

(defun vulpea-db-register-extractor (extractor-or-name &optional fn)
  "Register an extractor.

EXTRACTOR-OR-NAME can be:
- A `vulpea-extractor' struct (recommended)
- A symbol NAME with FN function (backward compatible)

When using the struct form:
  (vulpea-db-register-extractor
   (make-vulpea-extractor
    :name \\='my-extractor
    :version 1
    :priority 50
    :extract-fn #\\='my-extract-fn))

When using the simple form (backward compatible):
  (vulpea-db-register-extractor \\='my-extractor #\\='my-extract-fn)

FN should be a function taking (ctx note-data) where:
- ctx is a `vulpea-parse-ctx' structure; its AST slot is nil unless
  the extractor declares :requires-ast t (the simple form cannot,
  so AST readers must use the struct form)
- note-data is the plist being built for the note

FN should return updated note-data plist.  Changes it makes to core
note-data fields (:tags, :links, :meta, ...) are persisted: the
materialized notes row is updated and the normalized tables
re-synced after all extractors have run.  Returning nil means \"use
note-data as it now stands\" - in-place mutations included."
  (let ((extractor
         (cond
          ;; New struct form
          ((vulpea-extractor-p extractor-or-name)
           extractor-or-name)
          ;; Old simple form (backward compatible)
          ((and (symbolp extractor-or-name) fn)
           (make-vulpea-extractor
            :name extractor-or-name
            :extract-fn fn))
          (t
           (error "Invalid extractor: %S" extractor-or-name)))))
    ;; Validate extractor
    (unless (vulpea-extractor-name extractor)
      (error "Extractor must have a :name"))
    (unless (vulpea-extractor-extract-fn extractor)
      (error "Extractor must have an :extract-fn"))
    ;; Apply schema if present
    (vulpea-db--apply-plugin-schema extractor)

    ;; Remove existing extractor with same name
    (setq vulpea-db--extractors
          (cl-remove (vulpea-extractor-name extractor)
                     vulpea-db--extractors
                     :key #'vulpea-extractor-name))
    ;; Add new extractor
    (push extractor vulpea-db--extractors)
    ;; Sort by priority (lower priority runs first)
    (setq vulpea-db--extractors
          (sort vulpea-db--extractors
                (lambda (a b)
                  (< (vulpea-extractor-priority a)
                     (vulpea-extractor-priority b)))))
    ;; A running extraction worker mirrors the extractor registry;
    ;; keep it in sync (fboundp: vulpea-db-worker requires this file)
    (when (fboundp 'vulpea-db-worker-refresh-settings)
      (vulpea-db-worker-refresh-settings))
    extractor))

(defun vulpea-db-unregister-extractor (name)
  "Unregister extractor with NAME."
  (setq vulpea-db--extractors
        (cl-remove name vulpea-db--extractors
                   :key #'vulpea-extractor-name)))

(defun vulpea-db-get-extractor (name)
  "Get registered extractor by NAME.

Returns `vulpea-extractor' struct or nil if not found."
  (cl-find name vulpea-db--extractors
           :key #'vulpea-extractor-name))

(defun vulpea-db--run-extractors (ctx note-data)
  "Run all registered extractors on NOTE-DATA with CTX.

Extractors are run in priority order (lower priority first).
Returns updated note-data after all extractors have run; the caller
persists changes to core fields (see
`vulpea-db--insert-note-from-plist').  An extractor that returns nil
is treated as returning note-data as it now stands - in-place
mutations included.  The historical contract discarded return
values entirely, so plugins ending in a `when'-guarded insert must
stay harmless.

Only extractors declaring :requires-ast t see the AST slot of CTX;
every other extractor receives a copy whose AST is nil.  Handing
them whatever tree the core parse happened to produce would be
non-deterministic - sometimes the full object tree, sometimes the
degraded element-granularity one, and nil when results arrive from
the async worker.  An AST reader that forgot the declaration then
fails visibly on first test instead of subtly missing inline
objects."
  (let (ast-free-ctx)
    (cl-reduce
     (lambda (data extractor)
       (or (funcall (vulpea-extractor-extract-fn extractor)
                    (cond
                     ((vulpea-extractor-requires-ast-p extractor) ctx)
                     ((null (vulpea-parse-ctx-ast ctx)) ctx)
                     (t (or ast-free-ctx
                            (setq ast-free-ctx
                                  (let ((copy (copy-vulpea-parse-ctx ctx)))
                                    (setf (vulpea-parse-ctx-ast copy) nil)
                                    copy)))))
                    data)
           data))
     vulpea-db--extractors
     :initial-value note-data)))

;;; Update File

(defvar vulpea-db-note-index-filter-functions nil
  "Abnormal hook to veto notes before they are indexed.

Each function is called with the `vulpea-note' that is about to be
inserted while `vulpea-db-update-file' processes a file, and should
return non-nil to allow indexing it or nil to skip it.  A note is
indexed only when every function allows it
\(`run-hook-with-args-until-failure').  Functions may also emit warnings.

Handlers see the note built from the file currently being synced, so
they observe the live on-disk content.  This is the supported way to
react to a note as it enters the database - for example to surface
schema violations (see `vulpea-db-schema-validation-action').")

(defun vulpea-db--note-from-data (data path level)
  "Build a `vulpea-note' from extraction DATA for PATH at LEVEL.
Only the fields needed to reason about a note (identity, tags, meta,
links) are populated.  Used to present the note to
`vulpea-db-note-index-filter-functions' before insertion."
  (make-vulpea-note
   :id (plist-get data :id)
   :path path
   :level level
   :title (plist-get data :title)
   :tags (plist-get data :tags)
   :aliases (plist-get data :aliases)
   :meta (plist-get data :meta)
   :links (plist-get data :links)
   :properties (plist-get data :properties)
   :file-title (plist-get data :file-title)))

(defun vulpea-db--note-allowed-p (data path level)
  "Return non-nil when the note in DATA (at PATH, LEVEL) may be indexed.
Runs `vulpea-db-note-index-filter-functions'; with no handlers the note
is always allowed and no note object is built."
  (or (null vulpea-db-note-index-filter-functions)
      (run-hook-with-args-until-failure
       'vulpea-db-note-index-filter-functions
       (vulpea-db--note-from-data data path level))))

(defun vulpea-db-update-file (path)
  "Parse and update database for file at PATH.

Returns number of notes updated (file-level + headings)."
  (let* ((t0 (current-time))
         (ctx (vulpea-db--parse-file path))
         (t1 (current-time))
         (parse-time (* 1000 (float-time (time-subtract t1 t0)))))
    ;; Accumulate parse timing
    (when vulpea-db--timing-data
      (let ((parse-entry (assoc 'parse vulpea-db--timing-data)))
        (if parse-entry
            (setcdr parse-entry (+ (cdr parse-entry) parse-time))
          (push (cons 'parse parse-time) vulpea-db--timing-data))))
    (vulpea-db--apply-parse-ctx ctx)))

(defvar vulpea-db--org-id-files-seen (make-hash-table :test 'equal)
  "Session shadow of paths vulpea has pushed onto `org-id-files'.
`org-id-files' is a list; a `member' scan per registered file is
quadratic over a full rebuild.  The shadow makes the membership check
O(1).  It is conservative: cleared whenever `org-id-files' is
observed empty (e.g. after an external reset), and a stale positive
merely leaves a path unlisted - id lookups go through
`org-id-locations' regardless.")

(defun vulpea-db--register-id-locations (ids path)
  "Register note IDS at PATH with org-id, batched.

Equivalent to calling `org-id-add-location' for each ID, minus the
repeated path abbreviation and `org-id-files' scans that would do."
  (when (and ids org-id-track-globally)
    (unless org-id-locations (org-id-locations-load))
    ;; Ensure org-id-locations is a hash table first — during
    ;; org-id-update-id-locations it is temporarily an alist, and a
    ;; vulpea timer firing in that window would hit puthash on the
    ;; alist, causing "Wrong type argument: hash-table-p".
    (when (and org-id-locations (not (hash-table-p org-id-locations)))
      (setq org-id-locations (org-id-alist-to-hash org-id-locations)))
    (when (null org-id-files)
      (clrhash vulpea-db--org-id-files-seen))
    (let ((afile (abbreviate-file-name path)))
      (dolist (id ids)
        (puthash id afile org-id-locations))
      (unless (gethash afile vulpea-db--org-id-files-seen)
        (unless (member afile org-id-files)
          (push afile org-id-files))
        (puthash afile t vulpea-db--org-id-files-seen)))))

(defun vulpea-db--apply-parse-ctx (ctx &optional skip-org-id)
  "Write extraction results from CTX to the database.

CTX is a `vulpea-parse-ctx'.  Only its extracted data is required -
file node, heading nodes, hash, mtime, size, path; the AST may be
nil (extractor plugins, which do need the AST, are run against
whatever CTX carries).  This is the single write path shared by
synchronous updates and results arriving from the extraction worker.

Deletes the file's previous notes, inserts the new ones (honoring
`vulpea-db-note-index-filter-functions'), updates the stored file
hash, and registers note IDs with org-id - unless SKIP-ORG-ID is
non-nil, for callers that own no org-id state (the extraction
worker registers IDs in the main process instead).

Returns number of notes written (file-level + headings)."
  (let* ((path (vulpea-parse-ctx-path ctx))
         (db (vulpea-db))
         (count 0)
         (ids nil)  ; Track IDs to register with org-id
         (t0 (current-time))
         (db-time 0))

    (emacsql-with-transaction db
      ;; Delete existing notes from this file
      (vulpea-db--delete-file-notes path)

      ;; Insert file-level note
      (let ((file-data (vulpea-parse-ctx-file-node ctx)))
        (when-let* ((id (plist-get file-data :id)))
          (when (vulpea-db--note-allowed-p file-data path 0)
            (vulpea-db--insert-note-from-plist ctx path 0 0 file-data)
            (push id ids)
            (setq count (1+ count)))))

      ;; Insert heading-level notes
      (dolist (heading-data (vulpea-parse-ctx-heading-nodes ctx))
        (when (vulpea-db--note-allowed-p
               heading-data path (plist-get heading-data :level))
          (vulpea-db--insert-note-from-plist
           ctx
           path
           (plist-get heading-data :level)
           (plist-get heading-data :pos)
           heading-data)
          (when-let* ((id (plist-get heading-data :id)))
            (push id ids))
          (setq count (1+ count))))

      ;; Update file hash
      (vulpea-db--update-file-hash path
                                   (vulpea-parse-ctx-hash ctx)
                                   (vulpea-parse-ctx-mtime ctx)
                                   (vulpea-parse-ctx-size ctx)))
    (setq db-time (* 1000 (float-time (time-subtract (current-time) t0))))

    ;; Register all IDs with org-id so links can be followed
    (unless skip-org-id
      (vulpea-db--register-id-locations ids path))

    ;; Accumulate db timing
    (when vulpea-db--timing-data
      (let ((db-entry (assoc 'db vulpea-db--timing-data)))
        (if db-entry
            (setcdr db-entry (+ (cdr db-entry) db-time))
          (push (cons 'db db-time) vulpea-db--timing-data))))

    count))

(defconst vulpea-db--extractor-persisted-fields
  '(:title :properties :tags :aliases :meta :links :todo :priority
    :scheduled :deadline :closed :outline-path :attach-dir :file-title)
  "Note-data fields whose extractor-made changes are persisted.
When an extractor changes one of these in the note-data plist, the
change is written back to the notes row (and, for fields with a
normalized table, its rows) after all extractors have run.  Identity
fields (:id, :path, :level, :pos) are excluded - plugin tables hold
foreign keys against them.")

(defun vulpea-db--insert-note-from-plist (ctx path level pos data)
  "Insert note from DATA plist at PATH with LEVEL and POS.

CTX is the parse context containing AST and other metadata.
Runs registered extractors after insertion - the note goes in first
so extractor tables can hold foreign keys against it - then persists
any changes they made to core note-data fields
\(`vulpea-db--extractor-persisted-fields'): the materialized notes
row is updated and the normalized tables (tags, links, meta,
properties) re-synced, so plugin contributions to core fields behave
exactly like extracted ones.  Costs nothing when no extractors are
registered."
  (let* ((modified-at (format-time-string "%Y-%m-%d %H:%M:%S"
                                          (vulpea-parse-ctx-mtime ctx)))
         (properties (plist-get data :properties))
         (created-at (vulpea-db--extract-created-date properties)))
    ;; First insert the note so foreign keys can reference it
    (vulpea-db--insert-note
     :id (plist-get data :id)
     :path path
     :level level
     :pos pos
     :title (plist-get data :title)
     :properties properties
     :tags (plist-get data :tags)
     :aliases (plist-get data :aliases)
     :meta (plist-get data :meta)
     :links (plist-get data :links)
     :todo (plist-get data :todo)
     :priority (plist-get data :priority)
     :scheduled (plist-get data :scheduled)
     :deadline (plist-get data :deadline)
     :closed (plist-get data :closed)
     :outline-path (plist-get data :outline-path)
     :attach-dir (plist-get data :attach-dir)
     :file-title (plist-get data :file-title)
     :created-at created-at
     :modified-at modified-at)

    ;; Then run extractors that may insert into foreign-keyed tables
    (when vulpea-db--extractors
      ;; Snapshot core fields first: extractors mutate DATA in place
      ;; via plist-put, so the diff must compare against copies.  The
      ;; id is snapshotted too - the writeback must target the note
      ;; as inserted even if an extractor rewrites :id in place
      (let* ((id (plist-get data :id))
             (before (mapcar (lambda (field)
                               (copy-tree (plist-get data field)))
                             vulpea-db--extractor-persisted-fields))
             (updated (vulpea-db--run-extractors ctx data))
             (changes nil))
        (cl-loop for field in vulpea-db--extractor-persisted-fields
                 for old in before
                 for new = (plist-get updated field)
                 unless (equal old new)
                 do (push (cons field new) changes))
        ;; created-at derives from :properties; keep it in step
        (when (assq :properties changes)
          (let ((new-created-at (vulpea-db--extract-created-date
                                 (plist-get updated :properties))))
            (unless (equal new-created-at created-at)
              (push (cons :created-at new-created-at) changes))))
        (when changes
          (vulpea-db--update-note-fields id changes))))))

;;; Provide

(provide 'vulpea-db-extract)
;;; vulpea-db-extract.el ends here
