---
-- SYNOPSIS
-- ========
--
-- **pandoc** **-L** *pandoc-zotxt.lua* **-C**
--
-- DESCRIPTION
-- ===========
--
-- **pandoc-zotxt.lua** is a Lua filter for Pandoc that looks up bibliographic
-- data for citations in Zotero and adds that data to the "references"
-- metadata field or to a bibliography file, where Pandoc can pick it up.
--
-- You cite your sources using so-called "Better BibTeX citation keys"
-- (provided by Better BibTeX for Zotero) or "Easy Citekeys" (provided by
-- zotxt) and then tell **pandoc** to filter your document through
-- **pandoc-zotxt.lua** before processing citations. That's all there is to it.
--
-- For example:
--
--     pandoc -L pandoc-zotxt.lua -C <<EOF
--     See @DoeTitle2020 for details.
--     EOF
--
-- If the "references" metadata field or a bibliography file already contains
-- bibliographic data for a citation, that citation will *not* be looked up.
--
-- CONNECTING TO ZOTERO
-- ====================
--
-- Desktop client
-- --------------
--
-- By default, bibliographic data is fetched from the Zotero desktop client,
-- which must be running when you invoke **pandoc**. This is faster, easier,
-- and less error-prone than using Zotero's Web API. But it requires the zotxt
-- and BetterBibTeX for Zotero add-ons.
--
-- Web API
-- -------
--
-- Bibliographic data can also be fetched from the Zotero Web API.
--
-- To fetch data from your personal library, create a Zotero API key and set
-- the "zotero-api-key" metadata field to that key:
--
--     pandoc -L pandoc-zotxt.lua -C <<EOF
--     ---
--     zotero-connectors: zoteroweb
--     zotero-api-key: MO2GHxbkLnWgCqPtpoewgwIl
--     ...
--     Look up @DoeTitle2020 via the Zotero Web API.
--     EOF
--
-- You can also fetch bibliographic data from public Zotero groups. To do so,
-- list the IDs of those groups in the metadata field "zotero-public-groups";
-- fetching data from public groups does not require an API key.
--
--     pandoc -L pandoc-zotxt.lua -C <<EOF
--     ---
--     zotero-connectors: zoteroweb
--     zotero-public-groups:
--         - 199
--         - 4532986
--     ...
--     Look up @DoeTitle2020 in the Zotero groupsma 199 and 4532986.
--     EOF
--
-- The Zotero Web API does *not* allow to search for citation keys other than
-- Zotero item IDs. Therefore, BetterBibTeX citation keys and Easy Citekeys
-- have to be translated into author names, title keywords, and publication
-- years. Better BibTeX citation keys are split up at uppercase letters and
-- the first as well as the last of each series of digits ("DoeTitle2020"
-- becomes "Doe", "Title", "2020"). Easy Citekeys are split up at the first
-- colon and the last digit ("doe:2020title" becomes "doe", "2020", "title").
-- Citation keys that cannot be translated into at least two search terms are
-- ignored.
--
-- If a search yields two or more items, you need to disambiguate them. If you
-- use BetterBibTeX, you may want to set its citation key format to something
-- along the lines of "[auth][year][shorttitle3_3]" to make collisions less
-- likely. Alternatively, you can add an item's citation key to its "extra"
-- field in Zotero. Zotero's "extra" field is a list of CSL key-value pairs;
-- keys and values are separated by colons (":"), key-value pairs by
-- linefeeds. Use either the key "Citation key" or the key "Citekey" to add a
-- citation key (e.g., "Citation key: DoeTitle2020"); case is insignificant.
-- If you use Better BibTeX for Zotero, you can add the citation key it has
-- generated by 'pinning' it.
--
-- Support for accessing group libraries via the Zotero Web API is limited.
-- They are only searched if no item in your personal library matches.
-- Morever, the "extra" field of items in group libraries is ignored.
--
-- CACHING SOURCES WITH A BIBLIOGRAPHY FILE
-- ========================================
--
-- Bibliographic data can be added to a bibliography file, rather than to the
-- "references" metadata field. This speeds up subsequent processing, because
-- data that has already been fetched from Zotero need not be fetched again.
--
-- To use such a bibliography file, set the "zotero-bibliography" metadata
-- field to a filename. If that filename is relative, it is interpreted as
-- relative to the directory of the first input file or, if no input files
-- were given, the current working directory.
--
-- The filename may contain environment variables. Variable names must be
-- enclosed in `${...}`. They are replaced with the value of that variable
-- (e.g., `${HOME}` will be replaced with your home directory). Moreover, any
-- series of *n* dollar signs is replaced with *n* – 1 dollar signs, so
-- that you can escape them if they occur in the filename.
--
-- The format of the file is determined by its filename ending.
--
-- | **Ending** | **Format** |
-- | ---------- | ---------- |
-- | `.bib`     | BibLaTeX   |
-- | `.bibtex`  | BibTeX     |
-- | `.json`    | CSL JSON   |
-- | `.yaml`    | CSL YAML   |
--
-- Support for BibLaTeX and BibTeX files requires Pandoc v2.17 or later. CSL
-- is preferable to BibLaTeX and BibTeX.
--
-- The bibliography file is added to the "bibliography" metadata field
-- automatically; if that field already contains bibliography files, they take
-- priority.
--
-- Data is only ever added to the bibliography file, never updated or deleted.
-- However, if you delete the file, it will be regenerated from scratch.
--
-- For example:
--
--     pandoc -L pandoc-zotxt.lua -C <<EOF
--     ---
--     zotero-bibliography: ${HOME}/foo/bibliography.json
--     ...
--     See @DoeTitle2020 for details.
--     EOF
--
-- CITATION KEY TYPES
-- ==================
--
-- You can use citation keys of multitple types:
--
-- | **Name**          | **Type**          | **Example**   |
-- | ----------------- | ----------------- | ------------- |
-- | `betterbibtexkey` | Better BibTeX key | DoeTitle2020  |
-- | `easykey`         | Easy Citekey      | doe:2020title |
-- | `key`             | Zotero item ID    | A1BC23D4      |
--
-- You can force citation keys to only be interpreted as being of one of a
-- list of particular types by setting the "zotero-citekey-types" metadata
-- field:
--
--     pandoc -L pandoc-zotxt.lua -C <<EOF
--     ---
--     zotero-citekey-types: betterbibtexkey
--     ...
--     Force @DoeTitle to be treated as BetterBibTeX key.
--     EOF
--
-- This is useful if a Better BibTeX key is misinterpreted as Easy Citekey, or
-- vica vera.
--
-- CONFIGURATION
-- =============
--
-- You can configure how bibligraphic data is fetched by setting the following
-- metadata fields:
--
-- * `zotero-api-key`: A Zotero API key. Only applies to the '[Web
-- API](#web-api)'.
--
-- * `zotero-bibliography`: A bibliography filename. See '[Caching sources
-- with a bibliography file](#caching-sources-with-a-bibliography-file)' above.
--
-- * `zotero-citekey-types`: A list of citation key types. Citation keys are
-- interpreted to be of the listed types only. See '[Citation key
-- types](#citation-key-types)' above.
--
-- * `zotero-connectors`: One or more ways to connect to Zotero:
--
--   | **Key**   | **Fetch data from**   |
--   | --------- | --------------------- |
--   | zotxt     | Zotero desktop client |
--   | zoteroweb | Zotero Web API        |
--
--   Data is fetched via the given connectors in the order in which they are
--   given. If bibliographic data for a source can be fetched via an earlier
--   connector, it is *not* searched for via later ones. By default, data is
--   first searched for using zotxt and then using the Web API.
--
-- * `zotero-groups`: A list of Zotero group IDs. Only the given groups are
-- searched. By default, all groups you are a member of are searched. Only
-- applies to the '[Web API](#web-api)'.
--
-- * `zotero-public-groups`: A list of Zotero group IDs. The given groups are
-- searched in addition to non-public groups. Only applies to the '[Web
-- API](#web-api)'.
--
-- * `zotero-user-id`: A Zotero user ID. Looked up automatically if not given.
-- Only applies to the '[Web API](#web-api)'.
--
-- If a metadata field expects a list of values, giving a single item is the
-- same as giving a single-item list. For example:
--
--     pandoc -L pandoc-zotxt.lua -C <<EOF
--     ---
--     zotero-public-groups: 4532986
--     ...
--     See @DoeTitle2020 for details.
--     EOF
--
-- KNOWN ISSUES
-- ============
--
-- **pandoc-zotxt.lua** creates a temporary file when it adds bibliographic
-- data to a bibliography file. If Pandoc exits because it catches a signal
-- (e.g., because you press `Ctrl`-`c`), this file will *not* be deleted. This
-- is a bug in Pandoc (issue #7355) and in the process of being fixed.
-- Moreover, if you are using Pandoc up to v2.7, another process may,
-- mistakenly, use the same temporary file at the same time, though this is
-- highly unlikely.
--
-- A citation key may pick out the wrong item if it picks out a different
-- items depending on whether it is interpreted as a Better BibTeX key or as
-- an Easy Citekey. Set the 'zotero-citekey-types' metadata field to fix this
-- (see '[Citation key types](#citation-key-types)' above for details).
--
-- Zotero v5.0.71 and v5.0.72 fail to handle HTTP requests from user agents
-- that do not set the "User Agent" HTTP header. And **pandoc** does not. As a
-- consequence, **pandoc-zotxt.lua** cannot retrieve data from these versions
-- of Zotero unless you tell **pandoc** to set that header.
--
-- SECURITY
-- ========
--
-- If you are using Pandoc up to v2.7 and place the auto-generated
-- bibliography file in a directory that other users have write access to,
-- those users can read and change the content of that file, regardless of
-- whether they have permission to read or write the file itself.
--
-- SEE ALSO
-- ========
--
-- * [Zotero](https://www.zotero.org)
-- * [zotxt](https://github.com/egh/zotxt)
-- * [Better BibTeX for Zotero](https://retorque.re/zotero-better-bibtex/)
--
-- pandoc(1)
--
-- @script pandoc-zotxt.lua
-- @release 1.2.0
-- @author Odin Kroeger
-- @copyright 2018, 2019, 2020, 2021, 2022 Odin Kroeger
-- @license MIT

-- Table of Contents
--------------------

--- Metadata
-- @section Metadata

--- System
-- @section

--- Types
-- @section

--- Errors
-- @section

--- Variables
-- @section

--- Tables
-- @section

--- Strings
-- @section

--- Prototypes
-- @section

--- Warnings
-- @section

--- File I/O
-- @section

--- Networking
-- @section

--- Markup converters
-- @section

--- CSL items.
--
-- [Appendix IV](https://perma.cc/7LPL-F4XD) of the Citation Style Language
-- (CSL) specification lists all CSL variable names.
--
-- @section

--- Citation keys
-- @section

--- Bibliography files
-- @section

--- Document parsing
-- @section

--- Option parsing
-- @section

--- Zotero connectors
-- @section

--- Main
-- @section

-- Initialisation
-- ==============

-- luacheck: allow defined top
-- luacheck: globals PANDOC_SCRIPT_FILE PANDOC_STATE PANDOC_VERSION pandoc
-- luacheck: ignore DEBUG


-- Metadata
-- --------

-- The name of this script.
local NAME = 'pandoc-zotxt.lua'

-- The version of this script.
local VERSION = '1.2.0'


-- Custom global environment
-- -------------------------

-- Built-in functions.
local assert = assert
local error = error
local getmetatable = getmetatable
local next = next
local pairs = pairs
local pcall = pcall
local rawset = rawset
local require = require
local select = select
local setmetatable = setmetatable
local tonumber = tonumber
local tostring = tostring
local type = type

-- Modules.
local debug = debug
local io = io
local math = math
local os = os
local package = package
local string = string
local table = table
local utf8 = utf8

-- Pandoc.
local pandoc = pandoc
if not pandoc.utils then pandoc.utils = require 'pandoc.utils' end
if not pandoc.List then pandoc.List = require 'List' end

if pandoc.types then
    if PANDOC_VERSION >= {2, 8} and not pandoc.system then
        pandoc.system = require 'pandoc.system'
    end
    if PANDOC_VERSION >= {2, 12} and not pandoc.path then
        pandoc.path = require 'pandoc.path'
    end
end

local PANDOC_STATE = PANDOC_STATE
local PANDOC_SCRIPT_FILE = PANDOC_SCRIPT_FILE
local PANDOC_VERSION = PANDOC_VERSION

local M = {}
local _ENV = M


-- Libraries
-- ---------

do
    local repo = NAME .. '-' .. VERSION
    local path_sep = package.config:sub(1, 1)
    local script_dir = PANDOC_SCRIPT_FILE:match('(.*)' .. path_sep)
    local function path_join(...) return table.concat({...}, path_sep) end
    local sub_dir = path_join('share', 'lua', '5.4', '?.lua')
    package.path = table.concat({package.path,
        path_join(script_dir, sub_dir),
        path_join(script_dir, repo, sub_dir)
    }, ';')
end

local json = require 'lunajson'
local text = require 'text'
local pancake = require 'pancake'


-- Shorthands
-- ----------

local format = string.format
local concat = table.concat
local insert = table.insert
local pack = table.pack
local remove = table.remove
local unpack = table.unpack
local sort = table.sort
local char = utf8.char
local codes = utf8.codes

local stringify = pandoc.utils.stringify

local List = pandoc.List

local Meta = pandoc.Meta
local MetaInlines = pandoc.MetaInlines
local MetaList = pandoc.MetaList
local MetaMap = pandoc.MetaMap
local Pandoc = pandoc.Pandoc
local Plain = pandoc.Plain
local Str = pandoc.Str
local Span = pandoc.Span

local extend = List.extend
local find = List.find
local includes = List.includes
local map = List.map

local type_check = pancake.type_check
local type_match = pancake.type_match
local asserter = pancake.asserter
local protect = pancake.protect
local unprotect = pancake.unprotect
local copy = pancake.copy
local keys = pancake.keys
local sorted = pancake.sorted
local update = pancake.update
local xwarn = pancake.xwarn


---------
-- Tables
--
-- @section

--- A set representation.
--
-- Sets are represented as tables, so that a value `v` is a member of
-- a set `S` if and only if `S[v]` is truthy. See *Progamming in Lua*
-- ([chap. 11.5](https://www.lua.org/pil/11.5.html)) for details.
--
-- @caveats This table exists in documentation only.
-- @table Set


-------------
-- Networking
--
-- @section

--- Retrieve data from a URL via an HTTP GET request.
--
-- @string url A URL.
-- @treturn string The MIME type of the HTTP content, in lower case.
-- @treturn string The HTTP content itself.
-- @raise An error if the host cannot be connected to.
--  This error can only be caught in Pandoc ≥ v2.11.
--
-- @function http_get
http_get = type_check('string')(
    function (url)
        assert(url ~= '', 'URL is the empty string.')
        local mt, con = pandoc.mediabag.fetch(url, '.')
        return mt:lower(), con
    end
)


--------------------
-- Markup converters
--
-- @section

do
    -- Escape bold and italics meta characters.
    --
    -- @string chr The first character of a bold or italics expression.
    -- @string tail The rest of the expression.
    -- @treturn string The escaped expression.
    local function escape_bold_italics (chr, tail)
        return chr:gsub('(.)', '\\%1') .. tail
    end

    -- Escape superscript and subscript meta characters.
    --
    -- @string char The first character of a super- or subscript expression.
    -- @string body The content of the expression.
    -- @string tail The last character of the expression.
    -- @treturn string The escaped expression.
    local function escape_sup_sub (head, body, tail)
        return head:gsub('(.)', '\\%1') .. body .. tail:gsub('(.)', '\\%1')
    end

    -- Escape links and spans.
    --
    -- @string brack The bracketed expression.
    -- @string tail '(' or '{'.
    -- @treturn string The escaped expression.
    local function escape_link_span (brack, tail)
        return '\\[' .. brack:sub(2, -2) .. '\\]' .. tail
    end

    -- Pairs of expressions and replacement functions.
    local patterns = {
        -- Backslashes.
        {'(\\+)', '\\%1'},
        -- Bold and italics.
        -- Escapes liberally, but that's the easiest way to cover edge cases.
        {'(%*+)([^%s%*])', escape_bold_italics},
        {'(_+)([^%s_])', escape_bold_italics},
        -- Superscript and subscript.
        {'(%^+)([^%^%s]*)(%^+)', escape_sup_sub},
        {'(~+)([^~%s]+)(~+)', escape_sup_sub},
        -- Brackets (spans and links).
        {'(%b[])([%({])', escape_link_span}
    }

    local npatterns = #patterns

    --- Escape Markdown syntax.
    --
    -- @caveats
    --
    -- Only escapes [Markdown that Pandoc recognises in bibliographic
    -- data](https://pandoc.org/MANUAL.html#specifying-bibliographic-data).
    --
    -- @string str Non-Markdown text.
    -- @treturn string Text with Markdown syntax escaped.
    --
    -- @usage
    -- > escape_markdown '*text*'
    -- \*text*
    --
    -- @function escape_markdown
    escape_markdown = type_check('string')(
        function (str)
            for i = 1, npatterns do str = str:gsub(unpack(patterns[i])) end
            return str
        end
    )
end

do
    -- Shorthands
    -- luacheck: ignore type
    local type = pandoc.utils.type
    local walk_inline = pandoc.walk_inline
    local write = pandoc.write

    -- luacheck: ignore assert
    -- Don't add a stack trace.
    local assert = asserter()

    -- Filter to escape Markdown in Pandoc string elements.
    local escape_str = {}

    -- Escape Markdown in a string element.
    --
    -- Works like @{escape_markdown} but for Pandoc string elements.
    --
    -- @tparam pandoc.Str str A Pandoc string element.
    -- @treturn pandoc.Str A string with all Markdown syntax escaped.
    function escape_str.Str (str)
        return Str(escape_markdown(str.text))
    end

    -- Filter to convert to Markdown.
    local to_markdown = {}

    -- Make a function that converts an element to Markdown.
    --
    -- @string char A Markdown markup character.
    -- @treturn func A conversion function.
    local function converter (chr)
        return function (elem)
            local str = stringify(walk_inline(elem, to_markdown))
            return Str(chr .. str .. chr)
        end
    end

    -- Convert AST elements into Markdown text.
    to_markdown.Emph = converter '*'
    to_markdown.Strong = converter '**'
    to_markdown.Subscript = converter '~'
    to_markdown.Superscript = converter '^'

    local msg = '"%s" contains forbidden characters.'

    -- Convert HTML span elements to Markdown text.
    --
    -- @tparam pandoc.Span An HTML span element.
    -- @treturn pandoc.Str A Markdown representation.
    function to_markdown.Span (span)
        local str = stringify(pandoc.walk_inline(span, to_markdown))
        local attrs = {}
        local n = 0

        local identifier = span.identifier
        if identifier and identifier ~= '' then
            if identifier:match '%s' then error(msg:format(identifier), 0) end
            n = n + 1
            attrs[n] = '#' .. identifier
        end

        local classes = span.classes
        if classes then
            for i = 1, #classes do
                local class = classes[i]
                if class ~= '' then
                    if class:match '%s' then error(msg:format(class), 0) end
                    n = n + 1
                    attrs[n] = '.' .. class
                end
            end
        end

        local attributes = span.attributes
        if attributes then
            for k, v in pairs(attributes) do
                if v ~= '' then
                    if k:match '[%c%s="\'>/]' then error(msg:format(k), 0) end
                    if v:match '["\'=<>`]' then error(msg:format(v), 0) end
                    n = n + 1
                    attrs[n] = format('%s="%s"', k, v)
                end
            end
        end

        if n > 0 then
            str = '[' .. str ..                ']' ..
                  '{' .. concat(attrs, ' ') .. '}'
        end
        return Str(str)
    end

    -- Convert a Pandoc small capitals elements to Markdown text.
    --
    -- @tparam pandoc.SmallCaps A SmallCaps element.
    -- @treturn pandoc.Str A Markdown representation.
    function to_markdown.SmallCaps (sc)
        local span = Span(sc.content)
        insert(span.classes, 'smallcaps')
        return to_markdown.Span(span)
    end

    --- Convert a Pandoc element to Markdown text.
    --
    -- @require
    --
    -- Only recognises [elements Pandoc permits in bibliographic
    -- data](https://pandoc.org/MANUAL.html#specifying-bibliographic-data)
    -- if run with Pandoc < v2.17.
    --
    -- @tparam pandoc.AstElement elem A Pandoc AST element.
    -- @treturn string Markdown text.
    -- @raise An error if the element cannot be converted.
    --
    -- @function markdownify
    -- @todo Unit-test parse errors?
    markdownify_legacy = type_check('table|userdata')(
        function (elem)
            local escaped = pancake.elem_walk(elem, escape_str)
            return stringify(pancake.elem_walk(escaped, to_markdown))
        end
    )

    markdownify_modern = type_check('table|userdata')(
        function (elem)
            if type(elem) ~= 'Pandoc' then elem = Pandoc{Plain(elem)} end
            return write(elem, 'markdown'):gsub('\r?\n$', ''), nil
        end
    )

    if not pandoc.types or PANDOC_VERSION < {2, 17}
        then markdownify = markdownify_legacy
        else markdownify = markdownify_modern
    end
end

do
    local rep = string.rep
    local eol = pancake.EOL

    --- Create a number of spaces.
    --
    -- @int n The number of spaces.
    -- @treturn string `n` spaces.
    local function spaces (n)
        return rep(' ', n)
    end

    --- Convert a string to a YAML scalar.
    --
    -- @caveats
    --
    -- * Strings *must* be encoded in UTF-8.
    -- * Does *not* escape *all* non-printable characters.
    --
    -- @string str A string.
    -- @bool[opt] unquoted Quote strings only when needed?
    -- @treturn string A YAML scalar.
    local function scalarify (str, unquoted)
        -- Simple strings may need no special treatment.
        if
            unquoted                and
            not str:match '^%d+$'   and
            str:match '^[%w%s%-]+$'
        then return str end

        -- Replace line breaks with the OS' EOL sequence.
        str = str:gsub('\r?\n', eol)

        -- Escape special and control characters.
        local chars = {}
        local n = 0

        for _, c in codes(str, true) do
            if
                c == 0x22 or -- '"'
                c == 0x5c    -- '\'
            then
                n = n + 1
                chars[n] = '\\' .. char(c)
            elseif
                c == 0x09 or -- TAB
                c == 0x0a or -- LF
                c == 0x0d or -- CR
                c == 0x85    -- NEL
            then
                n = n + 1
                chars[n] = char(c)
            elseif
                c <= 0x001f or -- C0 control block
                c == 0x007f    -- DEL
            then
                n = n + 1
                chars[n] = format('\\x%02x', c)
            elseif
                (0x0080 <= c and c <= 0x009f) or -- C1 control block
                (0xd800 <= c and c <= 0xdfff) or -- Surrogate block
                c == 0xfffe or
                c == 0xffff
            then
                n = n + 1
                chars[n] = format('\\u%04x', c)
            else
                n = n + 1
                chars[n] = char(c)
            end
        end
        str = concat(chars)

        -- Quote.
        return '"' .. str .. '"'
    end

    -- Convert Lua to YAML types.
    local converters = {}
    converters.boolean = tostring
    converters.number = tostring
    converters.string = scalarify

    --- Generate a YAML representation of a value.
    --
    -- Terminates lines with @{pancake.EOL}.
    --
    -- @caveats
    --
    -- * Mangles strings in encodings other than UTF-8.
    -- * Does *not* escape *all* non-printable characters (because Unicode).
    --
    -- @param val A value.
    -- @int[opt=4] ind How many spaces to indent blocks.
    -- @treturn[1] string A YAML string.
    -- @raise An error if the value cannot be represented in YAML.
    --
    -- @function yamlify
    yamlify = type_check('?*', '?number', '?number', '?table')(
        function (val, ind, _col, _seen)
            if not _seen then _seen = {} end
            assert(not _seen[val], 'cycle in data tree.')
            if not ind then ind = 4 end
            assert(ind > 0, 'number of spaces must be greater than 0.')
            local t = type(val)
            local conv = converters[t]
            if conv then return conv(val) end
            assert(t == 'table', t .. ': cannot be expressed in YAML.')
            _seen[val] = true
            if not _col then _col = 0 end
            local nvals = #val
            local nkeys = select(2, keys(val))
            local sp = spaces(_col)
            local strs = {}
            local n = 0
            local function add (s, ...)
                n = n + 1
                strs[n] = s
                if ... then add(...) end
            end
            if nvals == nkeys then
                local col = _col + 2
                for i = 1, nvals do
                    local v = val[i]
                    if i > 1 then add(sp) end
                    add('- ', yamlify(v, ind, col, _seen))
                    if i ~= nvals then add(eol) end
                end
            else
                local i = 0
                for k, v in pairs(val) do
                    i = i + 1
                    if type(k) == 'number' then k = tostring(k)
                                           else k = scalarify(k, true)
                    end
                    if i > 1 then add(sp) end
                    add(k, ':')
                    local col = _col + ind
                    if type(v) == 'table' then add(eol, spaces(col))
                                          else add ' '
                    end
                    add(yamlify(v, ind, col, _seen))
                    if i ~= nkeys then add(eol) end
                end
            end
            return concat(strs)
        end
    )
end

--- Convert Zotero pseudo-HTML to proper HTML.
--
-- @string pseudo Zotero pseudo-HTML code.
-- @treturn[1] string HTML code.
-- @treturn[2] nil `nil` if opening and closing ´<sc>´ tags are unbalanced.
-- @treturn[2] string An error message.
--
-- @function zotero_to_html
zotero_to_html = type_check('string')(
    function (pseudo)
        local opened, closed, n, m
        opened, n = pseudo:gsub('<sc>', '<span class="smallcaps">')
        closed, m = opened:gsub('</sc>', '</span>')
        if n == m then return closed end
        return nil, format('encountered %d <sc> but %d </sc> tags.', n, m)
    end
)

do
    -- Shorthand.
    local read = pandoc.read

    --- Convert Zotero pseudo-HTML to Markdown.
    --
    -- @require
    --
    -- Only supports [pseudo-HTML that Pandoc recognises in bibliographic
    -- data](https://pandoc.org/MANUAL.html#specifying-bibliographic-data)
    -- if run with Pandoc < v2.17.
    --
    -- @string pseudo Zotero pseudo-HTML code.
    -- @treturn[1] string Markdown text.
    -- @treturn[2] nil `nil` if an error occurred.
    -- @treturn[2] string An error message.
    --
    -- @function zotero_to_markdown
    zotero_to_markdown = type_check('string')(protect(
        function (pseudo)
            local html, err = zotero_to_html(pseudo)
            if not html then return nil, err end
            local ok, result = pcall(read, html, 'html')
            if not ok then return nil, result end
            return markdownify(result)
        end
    ))
end


------------
-- CSL items
--
-- @section

--- Preferred order of CSL variables.
--
-- @see csl_vars_sort
CSL_VARS_ORDER = {
    'id',                       -- Item ID.
    'citekey',                  -- Citation key (CSL v1.2).
    'type',                     -- For example, 'paper', 'book'.
    'author',                   -- Author(s).
    'original-author',          -- Original author(s).
    'recipient',                -- Recipient of the document.
    'status',                   -- Publication status (e.g., 'forthcoming').
    'issued',                   -- When the item was published.
    'original-date',            -- Original date.
    'title',                    -- The title.
    'title-short',              -- A short version of the title.
    'short-title',              -- Ditto.
    'translator',               -- Translator(s).
    'editor',                   -- The editor(s).
    'container-title',          -- Publication the item was published in.
    'container-title-short',    -- A short version of that title.
    'collection-title',         -- E.g., a series.
    'collection-title-short',   -- A short version of the title.
    'edition',                  -- Container's edition.
    'volume',                   -- Volume no.
    'issue',                    -- Issue no.
    'page-first',               -- First page.
    'page',                     -- Pages or page range *or* number of pages.
    'publisher',                -- Publisher.
    'publisher-place',          -- City/cities the item was published in.
    'original-title',           -- Original title.
    'original-publisher',       -- Original publisher.
    'original-publisher-place', -- Place the item was originally published in.
    'url',                      -- The URL.
    'accessed',                 -- When the URL was last accessed.
    'doi',                      -- The DOI.
    'pmcid',                    -- PubMed Central reference number.
    'pmid',                     -- PubMed reference number.
    'isbn',                     -- The ISBN of the item.
    'issn',                     -- The ISSN of the container.
    'call-number',              -- Call number (of a library).
    'language',                 -- Language the item is in.
    'abstract',                 -- The abstract.
}

do
    --- Normalise a CSL key-value pair.
    --
    -- @string key A CSL key.
    -- @string val A value.
    -- @treturn[1] string A normalised CSL key.
    -- @treturn[1] string A trimmed value.
    -- @see csl_varname_normalise
    local function normalise (key, val)
        if not key or not val then return end
        key = csl_varname_normalise(key)
        if not key then return end
        return key, val:match '^%s*(.*%S)' or '', nil
    end

    -- Iterate over "extra" field entries.
    --
    -- Uses the newer `<name>: <value><linefeed>` syntax.
    --
    -- @string extra A Zotero "extra" field.
    -- @treturn func A *stateful* iterator.
    local function extras_modern (extra)
        local next_line = pancake.split(extra, '\r?\n')
        return function ()
            while true do
                local ln = next_line()
                while ln and ln:match '^%s*$' do ln = next_line() end
                if not ln then return end
                local k, v = normalise(ln:match '(.-)%s*:%s*(.*)')
                if k then return k, v end
            end
        end
    end

    -- Iterate over legacy "extra" field entries.
    --
    -- Uses the older `{:<name>: <value>}` syntax.
    --
    -- @caveats The syntax is poorly documented,
    --  it's unclear whether this function does the right thing.
    --
    -- @string extra A Zotero "extra" field.
    -- @treturn func A *stateful* iterator.
    local function extras_legacy (extra)
        local next_pair = extra:gmatch '{:([%a-]+):%s*([^}]+)}'
        return function ()
            while true do
                local k, v = next_pair()
                if not k then return end
                k, v = normalise(k, v)
                if k then return k, v end
            end
        end
    end

    --- Iterate over every key-value pair in the "note" field of a CSL item.
    --
    -- @caveats
    --
    -- Assumes that the "note" field is used to store entries that
    -- were added to Zotero's ["extra" field](https://perma.cc/EGN3-78CH).
    --
    -- @tab item A CSL item.
    -- @treturn func A *stateful* iterator.
    --
    -- @see csl_item_add_extras
    -- @function csl_item_extras
    csl_item_extras = type_check('table')(
        function (item)
            local extras = item.note
            if not extras then return function () return end end
            local iters = {extras_modern(extras), extras_legacy(extras)}
            local i = 1
            local iter = iters[i]
            return function ()
                while iter do
                    local k, v = iter()
                    if k then return k, v end
                    i = i + 1
                    iter = iters[i]
                end
            end
        end
    )
end

do
    local date_names = {'from', 'to'}
    local part_names = {'year', 'month', 'day'}

    -- Parse a date in Zotero's extra field.
    --
    -- @caveats The item is modified in-place.
    --
    -- @tab item A CSL item.
    -- @string key A field name.
    -- @string val A value.
    -- @raise An error if the date cannot be parsed.
    local function parse_date (item, key, val)
        -- luacheck: ignore assert
        local i, j
        local function msgh (err)
            return item.id or 'unknown item'
                   .. ': ' .. key
                   .. ': ' .. err:format(date_names[i], part_names[j])
        end
        local assert = asserter(nil, msgh)
        local date = {}
        i = 0
        for iso in val:gmatch '[^/]*' do
            i = i + 1
            assert(i < 3, 'too many dates.')
            local parts = {}
            j = 0
            for part in iso:gmatch '[^-]*' do
                if not part or part == '' then break end
                j = j + 1
                assert(j < 4, '%s date: too many date parts.')
                assert(part:match '^%d+$', '%s date: %s is not a number.')
                parts[j] = part
            end
            assert(j > 0, 'missing %s date.')
            date[i] = parts
        end
        assert(i > 0, 'is empty.')
        item[key] = {['date-parts'] = date}
    end

    -- Parse a name in Zotero's extra field.
    --
    -- @caveats The item is modified in-place.
    --
    -- @tab item A CSL item.
    -- @string key A field name.
    -- @string val A value.
    local function parse_name (item, key, val)
        local family, given = val:match '(.-)%s*||%s*(.*)'
        if not item[key] then item[key] = {} end
        if family and family ~= '' and given and given ~= ''
            then insert(item[key], {family = family, given = given})
            else insert(item[key], val)
        end
    end

    -- Mapping of CSL field names to functions that take an item and a field
    -- name-value pair and *add* that field to the CSL item *in-place*.
    local parsers = {
        ['accessed'] = parse_date,
        ['container'] = parse_date,
        ['event-date'] = parse_date,
        ['issued'] = parse_date,
        ['original-date'] = parse_date,
        ['submitted'] = parse_date,
        ['author'] = parse_name,
        ['collection-editor'] = parse_name,
        ['composer'] = parse_name,
        ['container-author'] = parse_name,
        ['director'] = parse_name,
        ['editor'] = parse_name,
        ['editorial-director'] = parse_name,
        ['managing-editor'] = parse_name,
        ['illustrator'] = parse_name,
        ['interviewer'] = parse_name,
        ['original-author'] = parse_name,
        ['recipient'] = parse_name,
        ['reviewed-author'] = parse_name,
        ['translator'] = parse_name
    }

    --- Add CSL variables from the "note" field to the item proper.
    --
    -- @side May print error messages to STDERR.
    --
    -- @tab item A CSL item.
    -- @treturn[1] table The item with variables from "extra" copied over.
    -- @treturn[2] nil `nil` if an "extra" field cannot be parsed.
    -- @treturn[2] string An error message.
    --
    -- @function csl_item_add_extras
    csl_item_add_extras = type_check('table')(protect(
        function (item)
            local ret = copy(item)
            for k, v in csl_item_extras(item) do
                if v == '' then
                    return nil, format('%s: %s: is empty.',
                                       item.id or 'unknown item', k)
                else
                    local f = parsers[k]
                    if not ret[k] or f == parse_date or k == 'type' then
                        if f then
                            f(ret, k, v)
                        -- At least until CSL v1.2 is out and
                        -- 'citekey' becomes official.
                        elseif k ~= 'citation-key' and k ~= 'citekey' then
                            ret[k] = v
                        end
                    end
                end
            end
            return ret
        end
    ))
end

do
    -- Shorthands
    local floor = math.floor

    -- Convert numbers to strings.
    --
    -- Floating point numbers are converted to integers.
    --
    -- @param num A number.
    -- @treturn[1] string A string.
    -- @treturn[2] nil `nil` if the given value is not a number.
    local function num_to_str (num)
        if type(num) == 'number' then return tostring(floor(num)) end
    end

    -- Normalise variable names of a CSL item.
    --
    -- @tab item A CSL item.
    -- @treturn tab A normalised item.
    -- @see csl_varname_normalise
    --
    -- @function csl_item_normalise_vars
    local function csl_item_normalise_vars (item, _seen)
        if     not _seen   then _seen = {}
        elseif _seen[item] then return _seen[item]
        end
        local ret = {}
        _seen[item] = ret
        for k, v in pairs(item) do
            if type(k) == 'string' then
                k = csl_varname_normalise(k)
            end
            if k then
                if type(v) == 'table' then
                    v = csl_item_normalise_vars(v, _seen)
                end
                ret[k] = v
            end
        end
        return ret
    end

    -- Normalise CSL items.
    --
    -- Normalises CSL variable names and numeric data.
    --
    -- @tparam tab item A CSL item.
    -- @treturn tab A normalised CSL item.
    csl_item_normalise = type_check('table')(
        function (item)
            return csl_item_normalise_vars(pancake.walk(item, num_to_str))
        end
    )
end

do
    -- A mapping of Lua type names to functions that construct Pandoc types.
    local converters = {}

    --- Convert a CSL Item to a Pandoc metadata type (worker).
    --
    -- @tab item CSL item.
    -- @param ... Passed on to per-type conversion function.
    -- @treturn pandoc.MetaValue A Pandoc metadata value.
    -- @raise An error if an item cannot be converted.
    local function convert (item, ...)
        local t = type(item)
        local f = converters[t]
        if f then return f(item, ...) end
        error(t .. ': cannot be converted to a Pandoc metadata value.', 0)
    end

    -- Convert a Lua boolean to a pandoc.MetaBool
    --
    -- @bool bool A boolean value.
    -- @treturn pandoc.MetaBool The value.
    function converters.boolean (bool)
        return pandoc.MetaBool(bool)
    end

    -- Convert a Lua number to a pandoc.MetaInlines string.
    --
    -- @bool num A number.
    -- @treturn pandoc.MetaInlines The number.
    function converters.number (num)
        return MetaInlines(List:new{Str(tostring(num))})
    end

    -- Convert Zotero pseudo-HTML to a pandoc.MetaInlines string.
    --
    -- @string str A string.
    -- @treturn pandoc.MetaInlines A string.
    -- @treturn[2] nil `nil` if a string cannot be converted.
    -- @treturn[2] string An error message.
    -- @raise An error if Zotero pseudo-HTML cannot be parsed.
    function converters.string (str)
        local html, err = zotero_to_html(str)
        if not html then error(err, 0) end
        local doc = pandoc.read(html, 'html')
        local inlines
        pcall(function () inlines = doc.blocks[1].content end)
        return MetaInlines(inlines or List:new())
    end

    -- Convert a Lua table to a pandoc.MetaMapping.
    --
    -- @tab tab A table.
    -- @treturn pandoc.MetaMapping The table.
    -- @fixme Recursion protection is not unit-tested.
    function converters.table (tab, _seen)
        if     not _seen  then _seen = {}
        elseif _seen[tab] then error 'cycle in data tree.'
        end
        _seen[tab] = true
        local n = select(2, keys(tab))
        if n == #tab then
            local list = MetaList{}
            for i = 1, n do list[i] = convert(tab[i], _seen) end
            return list
        end
        local mapping = MetaMap{}
        for k, v in pairs(tab) do mapping[k] = convert(v, _seen) end
        return mapping
    end

    --- Convert a CSL item to a Pandoc metadata value.
    --
    -- This allows to use CSL items in the `references` metadata field.
    --
    -- @tab item A CSL item.
    -- @treturn[1] pandoc.MetaValue A Pandoc metadata value.
    -- @treturn[2] nil `nil` if a value cannot be converted.
    -- @treturn[2] string An error message.
    --
    -- @function csl_item_to_meta
    -- @fixme Not unit-tested.
    csl_item_to_meta = type_check('table')(
        function (item)
            local ok, result = pcall(convert, item)
            if ok then return result end
            return nil, (item.id or 'unknown item') .. ': ' .. result
        end
    )
end

--- Filter CSL items by their citation key.
--
-- @side May print error messages to STDERR.
--
-- @tparam {tab,...} items CSL items.
-- @string ckey A citation key (e.g., `'doe:2020word'`, `'DoeWord2020'`).
-- @treturn {tab,...} Items with that citation key.
--
-- @function csl_items_filter_by_ckey
csl_items_filter_by_ckey = type_check('table', 'string')(
    function (items, ckey)
        local filt = {}
        local n = 0
        for i = 1, #items do
            local item = items[i]
            for k, v in csl_item_extras(item) do
                if k == 'citation-key' or k == 'citekey' then
                    if v == '' then
                        xwarn(item.id or 'unknown item', ': ${k}: is empty.')
                    elseif v:lower() == ckey:lower() then
                        n = n + 1
                        filt[n] = item
                        break
                    end
                end
            end
        end
        return filt
    end
)

--- Pick the IDs of CSL items out of a list of CSL items.
--
-- @side May print error messages to STDERR.
--
-- @tparam {tab,...} items CSL items.
-- @treturn Set Item IDs.
--
-- @function csl_items_ids
csl_items_ids = type_check('table')(
    function (items)
        local ids = {}
        for i = 1, #items do
            local id = items[i].id
            local t = type(id)
            if t == 'number'   then
                id = tostring(id)
            elseif t == 'userdata' or t == 'table' then
                id = stringify(id)
            end
            if type(id) == 'string' and id ~= ''
                then ids[id] = true
                else xwarn('@error', 'ignoring CSL item without parsable ID.')
            end
        end
        return ids
    end
)

--- Sort CSL items by their ID.
--
-- @caveats Assumes that CSL item IDs are of the same type.
--
-- @tab a A CSL item.
-- @tab b Another CSL item.
-- @treturn bool Whether item A comes before item B.
--
-- @function csl_items_sort
csl_items_sort = type_check('table', 'table')(
    function (a, b)
        return a.id < b.id
    end
)

--- Normalise a CSL variable name.
--
-- Trim the variable name, lowercase it, and replace spaces with dashes.
--
-- @string var A CSL variable name.
-- @treturn[1] string A normalised variable name.
-- @treturn[2] nil `nil` if the string is not a CSL variable name.
-- @treturn[2] string An error message.
--
-- @usage
-- > csl_varname_normalise 'Original date'
-- original-date
--
-- @function csl_varname_normalise
csl_varname_normalise = type_check('string')(
    function (var)
        var = var:match '^%s*(.*%S)' or ''
        if var == '' then
            return nil, 'variable name is the empty string.'
        elseif var:match '[^%a%s%-]' then
            return nil, var .. ': not a CSL variable.'
        end
        return var:gsub(' ', '-'):lower()
    end
)

--- Sort CSL variables.
--
-- Sorts variables in the order in which they are listed in @{CSL_VARS_ORDER}.
-- Unlisted variables are placed after listed ones in lexical order.
--
-- @string a A CSL variable name.
-- @string b Another CSL variable name.
-- @treturn bool Whether variable A should come before variable B.
--
-- @function csl_vars_sort
csl_vars_sort = type_check('string', 'string')(pancake.order(CSL_VARS_ORDER))


----------------
-- Citation keys
-- @section

--- A mapping of citation key types to citation key parsers.
--
-- A citation key parser is a function that takes a citation key and returns
-- the search terms and the number of search terms or `nil` if no search
-- terms can be derived from the key.
CITEKEY_PARSERS = {}

do
    local lower = text.lower

    -- Check whether a number represents a digit in ASCII.
    --
    -- @number n A number.
    -- @treturn bool Whether the number encodes a digit.
    local function is_digit (n)
        -- Digits start at 48 and end at 57 in ASCII.
        return 48 <= n and n <= 57
    end

    --- Guess search terms from a BetterBibTeX citation key.
    --
    -- Splits up a BetterBibTeX citation key at each uppercase letter
    -- and at each start of a string of digits.
    --
    -- @caveats Non-letter characters must be encoded in ASCII.
    --
    -- @string ckey A BetterBibTeX citation key (e.g., `'DoeWord2020'`).
    -- @treturn {string,...} Search terms.
    -- @treturn number Number of search terms.
    --
    -- @function CITEKEY_PARSERS.betterbibtexkey
    CITEKEY_PARSERS.betterbibtexkey = type_check('string')(
        function (ckey)
            assert(ckey ~= '', 'citation key is the empty string.')
            local sep = 1
            local prev
            local terms = {}
            local n = 0
            for pos, code in codes(ckey) do
                local add = false
                if pos > 1 then
                    if is_digit(code) ~= is_digit(prev) then
                        add = true
                    else
                        local chr = char(code)
                        -- `chr ~= lower(chr)` is `false` for non-letters,
                        -- so the string is only broken up at uppercase
                        -- letters, not at non-letters.
                        if chr ~= lower(chr) then
                            add = true
                        end
                    end
                end
                if add then
                    n = n + 1
                    terms[n] = ckey:sub(sep, pos - 1)
                    sep = pos
                end
                prev = code
            end
            n = n + 1
            terms[n] = ckey:sub(sep)
            return terms, n
         end
    )
end

--- Guess search terms from an Easy Citekey.
--
-- Splits up an Easy Citekey into an author, a year, and a word.
--
-- @string ckey A zotxt Easy Citekey (e.g., `'doe:2020word'`).
-- @treturn {string,...} Search terms.
-- @treturn number Number of search terms.
--
-- @function CITEKEY_PARSERS.easykey
CITEKEY_PARSERS.easykey = type_check('string')(
    function (ckey)
        assert(ckey ~= '', 'citation key is the empty string.')
        local pivot = 1
        local terms = {}
        local n = 0
        for pos, code in codes(ckey) do
            if pivot > 1 then
                -- Numbers range from 48 to 57 in ASCII/UTF-8.
                if 48 > code or code > 57 then
                    local year = ckey:sub(pivot, pos - 1)
                    if year and year ~= '' then
                        n = n + 1
                        terms[n] = year
                    end
                    local title = ckey:sub(pos)
                    if title and title ~= '' then
                        n = n + 1
                        terms[n] = title
                    end
                    return terms, n
                end
            -- The colon is encoded as 58 in ASCII/UTF-8.
            elseif code == 58 then
                local pre = ckey:sub(pivot, pos - 1)
                if pre and pre ~= '' then
                    n = n + 1
                    terms[n] = pre
                end
                pivot = pos + 1
            end
        end
        local year = ckey:sub(pivot)
        if year and year ~= '' then
            n = n + 1
            terms[n] = year
        end
        return terms, n
    end
)

--- A mapping of citation key type names to type guessing functions.
--
-- A type guessing function should take a citation key and return `true` if
-- that key *could be* (not "is") of that type and `false` otherwise.
CITEKEY_TYPIFIERS = {}

do
    -- Raise error if string contains non-printable or non-ASCII characters.
    --
    -- @string str The string.
    local function assert_printable_ascii (str)
        for _, c in codes(str) do assert(33 <= c and c <= 126) end
    end

    --- Check if a citation key could be a Better BibTeX citation key.
    --
    -- @string ckey A citation key.
    -- @treturn bool Whether the key could be a Better BibTeX key.
    --
    -- @function CITEKEY_TYPIFIERS.betterbibtexkey
    -- @fixme Not unit-tested.
    CITEKEY_TYPIFIERS.betterbibtexkey = type_check('string')(
        function (ckey)
            return pcall(assert_printable_ascii, ckey)
        end
    )
end

--- Check if a citation key is a Zotero item ID.
--
-- @string ckey A citation key.
-- @treturn bool Whether the citation key is a Zotero item ID.
--
-- @function CITEKEY_TYPIFIERS.key
-- @fixme Not unit-tested.
CITEKEY_TYPIFIERS.key = type_check('string')(
    function (ckey)
        return ckey:len() == 8 and ckey:match '^[%u%d]+$'
    end
)

do
    -- This makes sure that whether a key is an Easy Citekey is checked before
    -- it is checked whether it is a Better BibTeX key.
    local order = pancake.order{'easykey'}

    --- Guess search terms from a citation key.
    --
    -- @caveats The key must be supported by a parser in @{CITEKEY_PARSERS}.
    --
    -- @string ckey A citation key (e.g., `'doe:2020word'`, `'DoeWord2020'`).
    -- @tparam {string,...} types Types to try to parse the citation key as.
    -- @treturn[1] {string,...} Search terms.
    -- @treturn[2] nil `nil` if no search terms could be derived.
    -- @treturn[2] string An error message.
    --
    -- @function citekey_terms
    citekey_terms = type_check('string', 'table')(
        function (ckey, types)
            types = update({}, types)
            sort(types, order)
            for _, t in pairs(types) do
                local parse = CITEKEY_PARSERS[t]
                if parse then
                    local terms, n = parse(ckey)
                    if terms and n > 1 then return terms end
                end
            end
            return nil, format('cannot guess search terms for %s.', ckey)
        end
    )
end

--- Check whether a citation key could be of a type.
--
-- @string ckey A citation key.
-- @tparam {string,...} types Citation key types.
-- @treturn {string,...} Citation key types that would fit the key.
--
-- @function citekey_types
-- @fixme Not unit-tested.
citekey_types = type_check('string', 'table')(
    function (ckey, types)
        local ntypes = #types
        if ntypes == 1 then return types end
        local result = {}
        local n = 0
        for i = 1, ntypes do
            local t = types[i]
            local typify = CITEKEY_TYPIFIERS[t]
            if not typify or typify(ckey) then
                n = n + 1
                result[n] = t
            end
        end
        return result
    end
)


---------------------
-- Bibliography files
-- @section

--- A case-insensitive mapping of filename suffices to decoders.
--
-- A decoder is a function that takes a string and returns a list of CSL items.
BIBLIO_DECODERS = setmetatable({}, pancake.ignore_case)

--- Parse the contents of a BibLaTeX file.
--
-- @string str The contents of a BibLaTeX file.
-- @treturn[1] {tab,...} CSL items.
-- @treturn[2] nil `nil` if the contents could not be parsed.
-- @treturn[2] string An error message.
--
-- @require Only returns a list of mappings of the literal
--  'id' to CSL item IDs if run with Pandoc < v2.17.
--
-- @function BIBLIO_DECODERS.bib
-- @fixme Not unit-tested.
if not pandoc.types or PANDOC_VERSION < {2, 17} then
    BIBLIO_DECODERS.bib = type_check('string')(
        function (str)
            local ids = {}
            local n = 0
            for id in str:gmatch '@%w+%s*{%s*([^%s,]+)' do
                n = n + 1
                ids[n] = {id = id}
            end
            return ids
        end
    )
else
    BIBLIO_DECODERS.bib = type_check('string')(protect(
        function (str)
            local doc = pandoc.read(str, 'biblatex')
            local refs
            pcall(function () refs = doc.meta.references end)
            return refs or {}
        end
    ))
end

--- Parse the contents of a BibTeX file.
--
-- @string str The contents of a BibTeX file.
-- @treturn[1] {tab,...} CSL items.
-- @treturn[2] nil `nil` if the contents could not be parsed.
-- @treturn[2] string An error message.
--
-- @require Only returns a list of mappings of the literal
--  'id' to CSL item IDs if run with Pandoc < v2.17.
--
-- @function BIBLIO_DECODERS.bibtex
-- @fixme Not unit-tested.
if not pandoc.types or PANDOC_VERSION < {2, 17} then
    BIBLIO_DECODERS.bibtex = BIBLIO_DECODERS.bib
else
    BIBLIO_DECODERS.bibtex = type_check('string')(protect(
        function (str)
            local doc = pandoc.read(str, 'bibtex')
            local refs
            pcall(function () refs = doc.meta.references end)
            return refs or {}
        end
    ))
end

--- Parse a CSL JSON string.
--
-- @string str A CSL JSON string.
-- @treturn {tab,...} CSL items.
-- @treturn[2] nil `nil` if the string could not be parsed.
-- @treturn[2] string An error message.
--
-- @function BIBLIO_DECODERS.json
BIBLIO_DECODERS.json = type_check('string')(protect(
    function (str)
        local data = json.decode(str)
        assert(type(data) == 'table' and select(2, keys(data)) == #data,
               'not a list of CSL items.')
        return map(data, csl_item_normalise)
    end
))

--- Parse a CSL YAML string.
--
-- @caveats Converts formatting to Markdown, not Zotero pseudo-HTML.
--
-- @string str A CSL YAML string.
-- @treturn[1] {tab,...} CSL items.
-- @treturn[2] nil `nil` if the string could not be parsed.
-- @treturn[2] string An error message.
--
-- @function BIBLIO_DECODERS.yaml
BIBLIO_DECODERS.yaml = type_check('string')(protect(
    function (str)
        local next_line = str:gmatch '(.-)\r?\n'
        local ln = next_line(str, nil)
        while ln and ln ~= '---' do ln = next_line(str, ln) end
        if not ln then
            local eol = pancake.EOL
            str = concat{'---', eol, str, eol, '...', eol}
        end
        local doc = pandoc.read(str, 'markdown')
        local refs
        pcall(function () refs = doc.meta.references end)
        if not refs then return {} end
        return map(pancake.elem_walk(refs, {
            -- Pandoc ≥ v2.17.
            Blocks = markdownify,
            Inlines = markdownify,
            -- Pandoc < v2.17.
            MetaBlocks = markdownify,
            MetaInlines = markdownify
        }), csl_item_normalise)
    end
))

--- Alternative suffix for YAML files.
BIBLIO_DECODERS.yml = BIBLIO_DECODERS.yaml

--- A case-insensitive mapping of filename suffices to encoders.
--
-- @fixme Discuss codec protocol.
BIBLIO_ENCODERS = setmetatable({}, pancake.ignore_case)

--- Serialise a list of CSL items to a BibLaTeX string.
--
-- @tparam {tab,...} items CSL items.
-- @treturn[1] string A BibLaTeX string.
-- @treturn[2] nil `nil` if the items could not be serialised.
-- @treturn[2] string An error message.
--
-- @require Pandoc ≥ 2.17.
--
-- @function BIBLIO_ENCODERS.bib
-- @fixme Not unit-tested.
if pandoc.types and PANDOC_VERSION >= {2, 17} then
    BIBLIO_ENCODERS.bib = type_check('table')(protect(
        function (items)
            items = map(items, unprotect(csl_item_to_meta))
            local doc = Pandoc({}, Meta{references = items})
            return pandoc.write(doc, 'biblatex')
        end
    ))
end

--- Serialise a list of CSL items to a BibTeX string.
--
-- @tparam {tab,...} items CSL items.
-- @treturn[1] string A BibTeX string.
-- @treturn[2] nil `nil` if the items could not be serialised.
-- @treturn[2] string An error message.
--
-- @require Pandoc ≥ 2.17.
--
-- @function BIBLIO_ENCODERS.bibtex
-- @fixme Not unit-tested.
if pandoc.types and PANDOC_VERSION >= {2, 17} then
    BIBLIO_ENCODERS.bibtex = type_check('table')(protect(
        function (items)
            items = map(items, unprotect(csl_item_to_meta))
            local doc = Pandoc({}, Meta{references = items})
            return pandoc.write(doc, 'bibtex')
        end
    ))
end

--- Serialise a list of CSL items to a JSON string.
--
-- @tparam {tab,...} items CSL items.
-- @treturn[1] string A CSL JSON string.
-- @treturn[2] nil `nil` if the items could not be serialised.
-- @treturn[2] string An error message.
--
-- @function BIBLIO_ENCODERS.json
BIBLIO_ENCODERS.json = type_check('table')(protect(json.encode))

do
    -- Run walkers in protected mode.
    -- luacheck: ignore walk
    local walk = protect(pancake.walk)

    -- Run yamlify in protected mode.
    local yamlify = protect(yamlify)

    -- Convert Zotero pseudo-HTML to Markdown in CSL fields.
    --
    -- @param value A field value.
    -- @treturn[1] string Markdown text if the value contained pseudo-HTML.
    -- @treturn[2] nil `nil` if the value is not a `string`.
    -- @raise An error if a string cannot be parsed.
    local function to_markdown (value)
        if type(value) ~= 'string' then return end
        local md, err = zotero_to_markdown(value)
        if md then return md end
        error(format('cannot convert "%s" to Markdown: %s', value, err))
    end

    --- Serialise a list of CSL items to a YAML string.
    --
    -- @tparam {tab,...} items CSL items.
    -- @treturn[1] string A CSL YAML string.
    -- @treturn[2] nil `nil` if the items could not be serialised.
    -- @treturn[2] string An error message.
    --
    -- @function BIBLIO_ENCODERS.yaml
    BIBLIO_ENCODERS.yaml = type_check('table')(
        function (items)
            -- The Markdown writer appears to ignore metadata.
            for i = 1, #items do
                local item = items[i]
                local v, err = walk(item, to_markdown)
                if not v then
                    local id = item.id or 'unknown item'
                    return nil, id .. ': ' .. err
                end
                items[i] = v
            end
            return yamlify{references = items}
        end
    )
end

--- Alternative suffix for YAML files.
BIBLIO_ENCODERS.yml = BIBLIO_ENCODERS.yaml

--- Read a bibliography file.
--
-- The filename suffix determines the file's format.
-- @{BIBLIO_DECODERS} must contain a matching decoder.
--
-- @string fname A filename.
-- @treturn[1] tab CSL items.
-- @treturn[2] nil `nil` if an error occurred.
-- @treturn[2] string An error message.
-- @treturn[2] ?int An error number.
--
-- @function biblio_read
biblio_read = type_check('string')(protect(
    function (fname)
        -- luacheck: ignore assert
        assert(fname ~= '', 'filename is the empty string')
        local assert = asserter(nil, function (err)
            return pancake.path_prettify(fname) .. ': ' .. err
        end)
        local suffix = assert(fname:match '%.(%w+)$', 'no filename suffix')
        local decode = BIBLIO_DECODERS[suffix]
        assert(decode, 'unparsable file type.')
        local str, err, errno = pancake.file_read(fname)
        if not str then return nil, err, errno end
        local items = assert(decode(str))
        return items
    end
))

--- Write bibliographic data to a bibliography file.
--
-- The filename suffix determines the file's format.
-- @{BIBLIO_ENCODERS} must contain a matching encoder.
--
-- If no CSL items are given, tests whether data can be serialised
-- in the given format *without* chaging the file.
--
-- @caveats See @{file_write}.
--
-- @string fname A filename.
-- @tab[opt] items CSL items.
-- @treturn[1] string The filename suffix.
-- @treturn[2] nil `nil` if an error occurred.
-- @treturn[2] string An error message.
-- @treturn[2] ?int An error number.
--
-- @function biblio_write
biblio_write = type_check('string', '?table')(protect(
    function (fname, items)
        assert(fname ~= '', 'filename is the empty string')
        -- luacheck: ignore assert
        local assert = asserter(nil, function (err)
            return pancake.path_prettify(fname) .. ': ' .. err
        end)
        local suffix = assert(fname:match '%.(%w+)$', 'no filename suffix')
        local encode = BIBLIO_ENCODERS[suffix]
        assert(encode, 'unwritable file type.')
        if not items or #items == 0 then return suffix end
        local str = assert(encode(items))
        return pancake.file_write(fname, str, pancake.EOL)
    end
))

do
    local sort_csl_vars = {__pairs = sorted, sort = csl_vars_sort}

    --- Add new items from Zotero to a bibliography file.
    --
    -- @caveats See @{file_write}.
    -- @side May print error messages to STDERR.
    --
    -- @tparam connectors.Zotxt|connectors.ZoteroWeb handle A Zotero handle.
    -- @string fname The name of the bibliography file.
    -- @tparam {string,...} ckeys The citation keys of the items to add
    --  (e.g., `{'doe:2020word'}`).
    -- @treturn[1] bool `true` if the file was updated or no update was required.
    -- @treturn[2] nil `nil` if an error occurrs.
    -- @treturn[2] string An error message.
    -- @treturn[2] ?int An error number if the error is a file I/O error.
    -- @raise See @{connectors.Zotxt} and @{connectors.ZoteroWeb}.
    --
    -- @function biblio_update
    biblio_update = type_check('table', 'string', 'table')(protect(
        function (handle, fname, ckeys)
            -- luacheck: ignore fmt err errno
            assert(fname ~= '', 'filename is the empty string')
            if #ckeys == 0 then return true end
            local ok, err = biblio_write(fname)
            if not ok then return nil, err end
            local items, err, errno = biblio_read(fname)
            if not items then
                if errno ~= 2 then return nil, err, errno end
                items = {}
            end
            local ids = csl_items_ids(items)
            local n = #items
            for i = 1, #ckeys do
                local ckey = ckeys[i]
                if not ids[ckey] then
                    local item, err = handle:fetch(ckey)
                    if item then items[#items + 1] = item
                            else xwarn('@error', '@plain', err)
                    end
                end
            end
            if n == #items then return true end
            sort(items, csl_items_sort)
            for i = 1, #items do setmetatable(items[i], sort_csl_vars) end
            return biblio_write(fname, items)
        end
    ))
end


-------------------
-- Document parsing
--
-- @section

--- Collect bibliographic data.
--
-- Reads the `references` metadata field and every bibliography file.
--
-- @tparam pandoc.Pandoc doc A document.
-- @treturn[1] {tab,...} CSL items.
-- @treturn[2] nil `nil` if an error occurred.
-- @treturn[2] string An error message.
--
-- @function doc_srcs
doc_srcs_legacy = type_check('table|userdata')(protect(
    function (doc)
        -- luacheck: ignore assert
        local assert = asserter()
        local data = {}
        if not doc or not doc.meta then return data end
        local meta = doc.meta
        if meta.references then extend(data, meta.references) end
        if meta.bibliography then
            local opts = assert(pancake.opts_parse(meta, {
                name = 'bibliography',
                type = 'list'
            }))
            local fnames = opts.bibliography
            for i = 1, #fnames do
                -- luacheck: ignore err
                local fname = assert(pancake.file_locate(fnames[i]))
                local items = assert(biblio_read(fname))
                extend(data, items)
            end
        end
        return data
    end
))

-- @fixme The error messages of `pandoc.utils.references`
--        are sometimes unhelpful.
-- @fixme Rename with __
if not pandoc.types or PANDOC_VERSION < {2, 17}
    then doc_srcs = doc_srcs_legacy
    else doc_srcs = type_check('table|userdata')(
        protect(pandoc.utils.references)
    )
end

--- Collect the citation keys used in a document.
--
-- @tparam pandoc.Pandoc doc A document.
-- @bool[opt] undef Collect only undefind citation keys?
-- @treturn[1] {string,...} Citation keys.
-- @treturn[1] int The number of citation keys found.
-- @treturn[2] nil `nil` if an error occurred.
-- @treturn[2] string An error message.
--
-- @function doc_ckeys
doc_ckeys = type_check('table|userdata', '?boolean')(
    function (doc, undef)
        local meta = doc.meta or {}
        local blocks = doc.blocks
        local ckeys = {}
        local filter = {
            Cite = function (cite)
                ckeys = update(ckeys, csl_items_ids(cite.citations))
            end
        }
        for k, v in pairs(meta) do
            if k ~= 'references' then pancake.elem_walk(v, filter) end
        end
        for i = 1, #blocks do
            pandoc.walk_block(blocks[i], filter)
        end
        if undef then
            local srcs, err = doc_srcs(doc)
            if not srcs then return nil, err end
            for ckey in pairs(csl_items_ids(srcs)) do
                ckeys[ckey] = nil
            end
        end
        return keys(ckeys)
    end
)


--------------------
-- Zotero connectors
-- @section

--- A case-insensitive namespace for connector prototypes.
--
-- <h3>Connector protocol:</h3>
--
-- A connector prototype *must* provide two methods:
--
-- * `new`: Called to create handles.
--   Must have the same function signature as @{Object:new} and
--   return `nil` and an error message if the handle cannot be created.
--
-- * `fetch`: Called to fetch bibliographc data. Must take a citation key and
--   return either a CSL item or `nil` and an error message.
--
-- Connector prototypes may register configuration settings by providing
-- an `options` property that points to an @{pancake.Options} object.
-- That object's @{pancake.Options:parse} method is then called with the
-- document's metadata block as argument and whatever it returns is passed
-- on to `new`.
connectors = setmetatable({}, pancake.ignore_case)

--- Interface to [zotxt](https://github.com/egh/zotxt).
--
-- @usage
-- handle = connectors.Zotxt()
-- item = handle:fetch 'DoeWord2020'
--
-- @object connectors.Zotxt
-- @proto @{Object}
connectors.Zotxt = pancake.Object:clone()

--- Zotero options.
--
-- Defines `zotero-citekey-types`.
-- See the [manual](#content) for details.
--
-- @object connectors.Zotxt.options
-- @proto @{pancake.Options}.
connectors.Zotxt.options = pancake.Options{
    prefix = 'zotero',
    name = 'citekey_types',
    type = 'list'
}

--- Types of citation keys to expect.
--
-- List of @{string}s. For example, `{'betterbibtexkey', 'easykey'}`.
connectors.Zotxt.citekey_types = {
    'betterbibtexkey',  -- Better BibTeX citation key
    'easykey',          -- zotxt easy citekey
    'key',              -- Zotero item ID
}

do
    -- Shorthand.
    local decode = json.decode

    -- URL of the endpoint to look up items at.
    local base_url = 'http://localhost:23119/zotxt/items?%s=%s'

    --- Fetch a CSL item via zotxt.
    --
    -- @side May re-order the elements of @{connectors.Zotxt.citekey_types}.
    --
    -- @string ckey A citation key (e.g., `'doe:2020word'`, `'DoeWord2020'`).
    -- @treturn[1] table A CSL item.
    -- @treturn[2] nil `nil` if an error occurred.
    -- @treturn[2] string An error message.
    -- @raise See @{http_get}.
    --
    -- @function connectors.Zotxt:fetch
    connectors.Zotxt.fetch = type_check({citekey_types = 'table'}, 'string')(
        function (self, ckey)
            local ts = citekey_types(ckey, self.citekey_types)
            local err = ckey .. ': unrecognised type of citation key.'
            for i = 1, #ts do
                local t = ts[i]
                -- zotxt supports searching for multiple citation keys at
                -- once, but if a single one cannot be found, it replies
                -- with a cryptic error message (for Easy Citekeys) or an
                -- empty response (for Better BibTeX citation keys).
                local query_url = base_url:format(t, ckey)
                local ok, mt, str = pcall(http_get, query_url)
                if not ok then
                    error('failed to connect to Zotero desktop client.', 0)
                end
                if not mt or mt == '' or not str or str == '' then
                    if t == 'betterbibtexkey'
                        then err = ckey .. ': no matches.'
                        else err = ckey .. ': zotxt response is empty.'
                    end
                elseif not mt:match ';%s*charset="?utf%-?8"?%s*$' then
                    err = ckey .. ': zotxt response not encoded in UTF-8.'
                else
                    -- luacheck: ignore ok
                    local ok, data = pcall(decode, str)
                    if not ok then
                        err = 'zotxt responded: ' .. str
                    elseif type(data) == 'table' then
                        local n = #data
                        if n == 1 then
                            local citekey_types = self.citekey_types
                            if citekey_types[1] ~= t then
                                local j = select(2, find(citekey_types, t))
                                remove(citekey_types, j)
                                insert(citekey_types, 1, t)
                            end
                            local item = csl_item_normalise(data[1])
                            item.id = ckey
                            return item
                        end
                        if n == 0 then err = ckey .. ': no matches.'
                                  else err = ckey .. ': too many matches.'
                        end
                    elseif type(data) == 'string' then
                        err = 'zotxt responded: ' .. str
                    else
                        err = ckey .. ': cannot parse zotxt response: ' .. str
                    end
                end
            end
            return nil, err
        end
    )
end

--- Interface to [Zotero's Web API](https://www.zotero.org/support/dev/web_api)
--
-- @string[opt] api_key A Zotero Web API key.
-- @number[opt] user_id A Zotero user ID.
-- @tparam[opt] {number,...} groups Zotero groups to search in.
-- @tparam[opt] {number,...} public_groups Public Zotero groups to search in.
--
-- @usage
-- handle = connectors.ZoteroWeb{api_key = 'a0Bc1De2Fg3Hi4Jk5Lm6No7'}
-- item = handle:fetch 'DoeWord2020'
--
-- @object connectors.ZoteroWeb
-- @proto @{connectors.Zotxt}
connectors.ZoteroWeb = connectors.Zotxt:clone()

--- Types of citation keys to expect.
--
-- See @{connectors.Zotxt.citekey_types} for details
-- and @{citekey_terms} for caveats.
connectors.ZoteroWeb.citekey_types = List {
    'key',              -- Zotero item IDs
    'easykey',          -- zotxt Easy Citekey
    'betterbibtexkey',  -- Better BibTeX citation key
}

--- Zotero Web API options.
--
-- Defines:
--
--  * `zotero-citekey-types`
--  * `zotero-api-key`
--  * `zotero-user-id`
--  * `zotero-groups`
--  * `zotero-public-groups`
--
-- See the [manual](#content) for details.
--
-- @object connectors.ZoteroWeb.options
-- @proto @{pancake.Options}
connectors.ZoteroWeb.options = pancake.Options(
    {prefix = 'zotero', name = 'citekey_types', type = 'list'},
    {prefix = 'zotero', name = 'api_key'},
    {prefix = 'zotero', name = 'user_id', type = 'number'},
    {prefix = 'zotero', name = 'groups', type = 'list<number>'},
    {prefix = 'zotero', name = 'public_groups', type = 'list<number>'}
)

do
    -- Shorthand.
    local decode = json.decode

    -- Zotero Web API base URL.
    local base_url = 'https://api.zotero.org'

    -- URL template for user ID lookups.
    local user_id_url = base_url .. '/keys/${api_key}'

    -- URL template for group membership lookups.
    local groups_url = base_url .. '/users/${user_id}/groups'

    -- User prefix.
    local user_prefix = '/users/${user_id}'

    -- Group prefix.
    local group_prefix = '/groups/${group_id}'

    -- URL template for item lookups.
    local items_url = base_url .. '${prefix}/items/${id}'

    --- Metatable for Zotero Web API connectors.
    connectors.ZoteroWeb.mt = getmetatable(connectors.ZoteroWeb)

    --- Getters for Zotero Web API connectors.
    --
    -- @see getterify
    connectors.ZoteroWeb.mt.getters = {}

    --- Get the user ID for the given API key.
    --
    -- @tparam connectors.ZoteroWeb obj A Zotero Web API handle.
    -- @treturn string A Zotero user ID.
    -- @raise An error if
    --
    -- * the `api_key` field is not set.
    -- * the Zotero Web API could not be reached (see @{http_get} for details),
    -- * the API's response cannot be parsed, or
    -- * no user ID could be found for the given Zotero API key.
    --
    -- @function connectors.ZoteroWeb.mt.getters.user_id
    connectors.ZoteroWeb.mt.getters.user_id = type_check('table')(
        function (obj)
            -- luacheck: ignore err
            -- @fixme Test if this is ever raised!
            assert(obj.api_key, 'no Zotero API key given.')
            local ep = assert(pancake.vars_sub(user_id_url, obj))
            local data, err = obj.query(ep, {v = 3})
            if data then
                local user_id = data.userID
                if not user_id then
                    err = format('no user for API key %s.', obj.api_key)
                else
                    obj.user_id = user_id
                    return user_id
                end
            end
            error('Zotero user ID lookup: ' .. err, 0)
        end
    )

    --- Get the groups the user is a member of.
    --
    -- @tparam connectors.ZoteroWeb obj A Zotero Web API handle.
    -- @treturn {string,...} Zotero group IDs.
    -- @raise An error if
    --
    -- * the `api_key` field is not set.
    -- * the Zotero Web API could not be reached
    --   (see @{http_get} for details), or
    -- * the API's response cannot be parsed.
    --
    -- @function connectors.ZoteroWeb.mt.getters.groups
    connectors.ZoteroWeb.mt.getters.groups = type_check('table')(
        function (obj)
            -- luacheck: ignore err
            -- @fixme Is this ever raised?
            assert(obj.api_key, 'no Zotero API key given.')
            local ep = assert(pancake.vars_sub(groups_url, obj))
            local data, err = obj.query(ep, {v = 3, key = obj.api_key})
            if data then
                local groups = {}
                for i = 1, #data do
                    if data[i] and data[i].data and data[i].data.id then
                        groups[#groups + 1] = data[i].data.id
                    end
                end
                obj.groups = groups
                return groups
            end
            error('Zotero group membership lookup: ' .. err, 0)
        end
    )

    --- Delegate to the Zotero Web API interface.
    --
    -- @function connectors.ZoteroWeb.clone
    -- @see Object:clone
    -- @see getterify
    connectors.ZoteroWeb.clone = type_check('table')(
        function (...)
            return pancake.getterify(connectors.Zotxt.clone(...))
        end
    )

    --- Create a new Zotero Web API handle.
    --
    -- @fixme Discuss options.
    connectors.ZoteroWeb.new = type_check('table', {
        api_key = '?string',
        user_id = '?number',
        groups = '?table',
        public_groups = '?table'
    })(
        function (proto, args)
            if args.api_key or args.public_groups then
                return connectors.Zotxt.new(proto, args)
            end
            return nil, 'set an API key or public groups to use the Zotero Web API.'
        end
    )

    --- Retrieve data from the Zotero Web API.
    --
    -- @string ep An endpoint URL.
    -- @tparam {string=string,...} params Request parameters.
    -- @treturn[1] string The response of the Zotero Web API.
    -- @treturn[2] nil `nil` if an error occurred.
    -- @treturn[2] string An error message.
    -- @raise See @{http_get}.
    --
    -- @see http_get
    -- @function connectors.ZoteroWeb.query
    -- @fixme Not unit-tested.
    connectors.ZoteroWeb.query = type_check('string', '?table')(
        function (ep, params)
            local query_url = ep
            if params then
                local query = {}
                local n = 0
                for k, v in sorted(params) do
                    n = n + 1
                    query[n] = k .. '=' ..v
                end
                query_url = query_url .. '?' .. concat(query, '&')
            end
            local ok, mt, str = pcall(http_get, query_url)

            if not ok then error('failed to connect to Zotero Web API.', 0) end
            if not mt or mt == '' then
                return nil, 'Zotero response declares no MIME type.'
            elseif not str or str == '' then
                return nil, 'Zotero response is empty.'
            -- @todo Check encoding of response.
            elseif mt:match '^text/plain' then
                return nil, 'Zotero responded: ' .. str
            elseif not mt:match '%f[%a]json%f[%A]' then
                local err = 'Zotero response is of unexpected MIME type %s.'
                return nil, err:format(mt)
            end
            local data
            ok, data = pcall(decode, str)
            if not ok then
                return nil, 'cannot parse Zotero response: ' .. str
            end
            return data
            -- end
            -- return nil, 'Zotero responded: ' .. str
        end
    )

    --- Iterate over item endpoint URLs.
    --
    -- @string[opt] id A Zotero item ID.
    -- @treturn func A *stateful* iterator.
    --
    -- @function connectors.ZoteroWeb:endpoints
    connectors.ZoteroWeb.endpoints = type_check('table', '?string')(
        function (self, id)
            if not id then id = '' end
            local groups, ngroups
            local i = -1
            return function ()
                i = i + 1
                if i == 0 then
                    if self.api_key then
                        return assert(pancake.vars_sub(items_url, {
                            prefix = user_prefix,
                            user_id = self.user_id,
                            id = id
                        }))
                    end
                else
                    if not groups then
                        groups = {}
                        if self.api_key then
                            extend(groups, self.groups)
                        end
                        if self.public_groups then
                            extend(groups, self.public_groups)
                        end
                        ngroups = #groups
                    end
                    if i > ngroups then return end
                    return assert(pancake.vars_sub(items_url, {
                        prefix = group_prefix,
                        group_id = groups[i],
                        id = id
                    }))
                end
            end
        end
    )

    --- Search items by their author, publication year, and title.
    --
    -- @caveats Does *not* correct Zotero's CSL JSON export.
    --
    -- @string ... Search terms.
    -- @treturn[1] tab CSL items that match the given search terms.
    -- @treturn[2] nil `nil` if no items were found or an error occurred.
    -- @treturn[2] string An error message.
    -- @treturn[2] An error message.
    -- @raise See @{connectors.ZoteroWeb.mt.getters.user_id} and
    --  @{connectors.ZoteroWeb.mt.getters.groups}.
    --
    -- @function connectors.ZoteroWeb:search
    connectors.ZoteroWeb.search = type_check(
        {
            endpoints = 'function',
            query = 'function'
        },
        'string',
        '?string',
        '...'
    )(
        function (self, ...)
            local q = concat({...}, '+')
            local params = {v = 3, key = self.api_key,
                            q = q, qmode = 'titleCreatorYear',
                            format ='csljson', itemType='-attachment'}
            for ep in self:endpoints() do
                -- luacheck: ignore err
                local data, err = self.query(ep, params)
                if not data then return nil, err end
                local items = data.items
                if items and #items > 0 then
                    return map(items, csl_item_normalise)
                end
            end
            return nil, 'no matches.'
        end
    )

    --- Look up a CSL item by its Zotero ID.
    --
    -- @string id A Zotero item ID.
    -- @treturn[1] tab A CSL item.
    -- @treturn[2] nil `nil` if no or more than one item has been found.
    -- @treturn[2] string An error message.
    -- @raise See @{connectors.ZoteroWeb.mt.getters.user_id} and
    --  @{connectors.ZoteroWeb.mt.getters.groups}.
    --
    -- @function connectors.ZoteroWeb:lookup
    connectors.ZoteroWeb.lookup = type_check({
        endpoints = 'function',
        query = 'function'
    }, 'string')(
        function (self, id)
            local params = {v = 3, key = self.api_key,
                            format ='csljson', itemType='-attachment'}
            for ep in self:endpoints(id) do
                local data = self.query(ep, params)
                if data then
                    local items = data.items
                    if items then
                        local n = #items
                        if n == 1 then
                            -- luacheck: ignore err
                            local err
                            local item = csl_item_normalise(items[1])
                            item, err = csl_item_add_extras(item)
                            if not item then return id .. ': ' .. err end
                            item.id = id
                            return item
                        elseif n > 1 then
                            return nil, format('item ID %s: not unique.', id)
                        end
                    end
                end
            end
            return nil, id .. ': no matches.'
        end
    )

    --- Fetch a CSL item from the Zotero Web API.
    --
    -- @side
    --
    -- Search terms for citation keys are printed to STDERR
    -- if the user has requested verbose output.
    --
    -- @string ckey A citation key (e.g., `'doe:2020word'`, `'DoeWord2020'`).
    -- @treturn[1] table A CSL item.
    -- @treturn[2] nil `nil` if an error occurred.
    -- @treturn[2] string An error message.
    -- @raise See @{connectors.ZoteroWeb.mt.getters.user_id} and
    --  @{connectors.ZoteroWeb.mt.getters.groups}.
    --
    -- @function connectors.ZoteroWeb:fetch
    connectors.ZoteroWeb.fetch = type_check({
        citekey_types = 'table',
        lookup = 'function',
        search = 'function'
    }, 'string')(
        function (self, ckey)
            -- luacheck: ignore err
            assert(ckey ~= '', 'citation key is the empty string.')
            if includes(citekey_types(ckey, self.citekey_types), 'key') then
                return self:lookup(ckey)
            end
            local terms, err = citekey_terms(ckey, self.citekey_types)
            if not terms then return nil, err end
            xwarn('@info', '${ckey}: searching for: ',
                  '@plain', concat(terms, ', '))
            local items, err = self:search(unpack(terms))
            if not items then return nil, ckey .. ': ' .. err end
            local n = #items
            if n == 0 then
                return nil, ckey .. ': no matches.'
            elseif n > 1 then
                items = csl_items_filter_by_ckey(items, ckey)
                if not items or items.n == 0 then
                    return nil, ckey .. ': too many matches.'
                elseif items.n > 1 then
                    return nil, format('citation key %s: not unique.', ckey)
                end
            end
            local item = items[1]
            item.id = ckey
            local err
            item, err = csl_item_add_extras(item)
            if not item then return ckey .. ': ' .. err end
            return item
        end
    )
end


-------
-- Main
--
-- @section

--- Add bibliographic data to a bibliography file.
--
-- Updates the bibliography file as needed and adds its to the
-- `bibliography` metadata field. Interpretes relative filenames
-- as relative to the directory of the first input file passed
-- to **pandoc**, or, if no input files were given, as relative
-- to the current working directory.
--
-- @caveats @{file_write}.
-- @side May print error messages to STDERR.
--
-- @string fname A filename for the bibliography file.
-- @tparam connectors.Zotxt|connectors.ZoteroWeb handle A Zotero handle.
-- @tparam pandoc.Pandoc doc A Pandoc document.
-- @treturn[1] pandoc.Meta An updated metadata block.
-- @treturn[2] nil `nil` if nothing was done or an error occurred.
-- @treturn[2] string An error message, if applicable.
-- @raise See @{connectors.Zotxt} and @{connectors.ZoteroWeb}.
--
-- @function add_biblio
add_biblio = type_check('string', 'table', 'table|userdata')(
    function (fname, handle, doc)
        -- luacheck: ignore err
        assert(fname ~= '', 'bibliography filename is the empty string.')
        local ckeys, err = doc_ckeys(doc, true)
        if not ckeys then return nil, err end
        if #ckeys == 0 then return end
        local meta = doc.meta
        local dir = pancake.project_dir()
        assert(dir ~= '', 'directory of first input file is the empty string.')
        if not pancake.path_is_abs(fname) then
            fname = pancake.path_join(dir, fname)
        end
        local ok, err = biblio_update(handle, fname, ckeys)
        if not ok then return nil, err end
        local opts, err = pancake.opts_parse(meta, {
            name = 'bibliography',
            type = 'list'
        })
        if not opts then return nil, err end
        local fnames = opts.bibliography or List:new()
        fnames:insert(fname)
        meta.bibliography = fnames
        return meta
    end
)

--- Add bibliographic data to the `references` metadata field.
--
-- @side May print error messages to STDERR.
--
-- @tparam connectors.Zotxt|connectors.ZoteroWeb handle A Zotero handle.
-- @tparam pandoc.Pandoc doc A Pandoc document.
-- @treturn[1] table An updated metadata block.
-- @treturn[2] nil `nil` if nothing was done or an error occurred.
-- @treturn[2] string An error message, if applicable.
-- @raise See @{connectors.Zotxt} and @{connectors.ZoteroWeb}.
--
-- @function add_refs
add_refs = type_check('table', 'table|userdata')(protect(
    function (handle, doc)
        local ckeys, err = doc_ckeys(doc, true)
        if not ckeys then return nil, err end
        local meta = doc.meta
        if #ckeys == 0 then return end
        if not meta.references then meta.references = MetaList({}) end
        local n = #meta.references
        for i = 1, #ckeys do
            -- luacheck: ignore err
            local ckey = ckeys[i]
            local item, err = handle:fetch(ckey)
            if item then
                item, err = csl_item_to_meta(item)
                if item then n = n + 1
                             meta.references[n] = item
                        else xwarn('@error', '@plain', ckey, ': ', err)
                end
            else
                xwarn('@error', '@plain', err)
            end
        end
        return meta
    end
))

do
    local parser = pancake.Options(
        {
            prefix = 'zotero',
            name = 'bibliography',
            parse = function (fname)
                return pancake.env_sub(fname)
            end
        },
        {
            prefix = 'zotero',
            name = 'connectors',
            type = 'list',
            parse = function (names)
                local conns = {}
                for i = 1, #names do
                    local name = names[i]
                    if not name:match '^%a[%w_]+$' then
                        return nil, name .. ': not a connector name.'
                    end
                    local conn = connectors[name]
                    if not conn then
                        return nil, name .. ': no such connector.'
                    elseif not type_match(conn, {
                        new = 'function',
                        fetch = 'function'
                    }) then
                        return nil, name .. ': connector violates protocol.'
                    end
                    conns[#conns + 1] = conn
                end
                return conns
            end
        }
    )

    --- Collect citations and add bibliographic data to the document.
    --
    -- See the [manual](#content) for details.
    --
    -- @side May print error messages to STDERR.
    --
    -- @tparam pandoc.Pandoc doc A document.
    -- @treturn[1] pandoc.Pandoc The document with bibliographic data added.
    -- @treturn[2] nil `nil` if nothing was done or an error occurred.
    -- @treturn[2] ?string An error mesage, if applicable.
    -- @raise See @{connectors.Zotxt} and @{connectors.ZoteroWeb}.
    main = protect(
        function (doc)
            -- luacheck: ignore assert
            local assert = asserter()
            local opts = assert(parser:parse(doc.meta))
            local handles = {}

            local conns = opts.connectors
            if not conns or #conns == 0 then
                for _, conn in sorted(connectors, order{'zotxt'}) do
                    -- luacheck: ignore opts
                    local args
                    local opts = conn.options
                    if opts then args = assert(opts:parse(doc.meta)) end
                    local handle = conn:new(args)
                    if handle then handles[#handles + 1] = handle end
                end
            else
                for i = 1, #conns do
                    -- luacheck: ignore opts
                    local conn = conns[i]
                    local opts = conn.options
                    local args
                    if opts then args = assert(opts:parse(doc.meta)) end
                    local handle = assert(conn:new(args))
                    handles[#handles + 1] = handle
                end
            end

            local add_sources
            local biblio = opts.bibliography
            if biblio then
                function add_sources (...) return add_biblio(biblio, ...) end
            else
                add_sources = add_refs
            end

            local chg = false
            for i = 1, #handles do
                local meta, err = add_sources(handles[i], doc)
                if meta then
                    doc.meta = meta
                    chg = true
                elseif err then
                    xwarn('@error', '@plain', err)
                end
            end
            if chg then return doc end
        end
    )
end

-- Returning the whole script, rather than only a list of mappings of
-- Pandoc data types to functions, allows to do unit testing.

M[1] = {Pandoc = function (doc)
    local new, err = main(doc)
    if err then xwarn('@error', '@plain', err) end
    return new
end}

return M
