xquery version "3.1";

(:
 : This module deals with the tokenization of the relevant text by identifying
 : the relevant words and wrapping them in a separate tei:w.
 : This tei:w is equipped with a unique ID which contains
 :      - the manuscript's IDNO
 :      - the node ID of the complete text node and
 :      -  the position of the word in focus within the text node
 :
 : This way, each word is specifically addressable.
 :)

module namespace tokenize="http://ahiqar.uni-goettingen.de/ns/tokenize";

declare namespace tei="http://www.tei-c.org/ns/1.0";

declare function tokenize:main($TEI as element(tei:TEI))
as element(tei:TEI) {
    let $id-prefix := tokenize:make-id-from-idno($TEI)
    let $enhanced-texts :=
        for $text in $TEI//tei:group/tei:text return
            tokenize:add-ids($text, $id-prefix)
    return
        element {QName("http://www.tei-c.org/ns/1.0", "TEI")} {
            $TEI/tei:teiHeader,
            element {QName("http://www.tei-c.org/ns/1.0", "text")} {
                element {QName("http://www.tei-c.org/ns/1.0", "group")} {
                    $enhanced-texts
                }
            }
        }
};

declare function tokenize:add-ids($nodes as node()*,
    $id-prefix as xs:string)
as node()* {
    for $node in $nodes return
        typeswitch ($node)
        case text() return
            if (tokenize:is-text-relevant($node)) then
                tokenize:add-id-to-text($node, $id-prefix)
            else
                $node

        case comment() return
            ()

        case processing-instruction() return
            $node

        default return
            element {QName($node/namespace-uri(), $node/local-name())} {
                $node/@*,
                tokenize:add-ids($node/node(), $id-prefix)
            }
};

declare function tokenize:is-text-relevant($text as text())
as xs:boolean {
    if($text[not(ancestor::tei:sic)]
            [not(ancestor::tei:surplus)]
            [not(ancestor::tei:supplied)]
            [not(ancestor::tei:*[@type = "colophon"])]
            [not(ancestor::tei:g)]
            [not(ancestor::tei:unclear)]
            [not(ancestor::tei:catchwords)]
            [not(ancestor::tei:bibl)]
            [not(ancestor::tei:note) or ancestor::tei:quote]) then
        true()
    else
        false()
};

declare function tokenize:add-id-to-text($text as text(),
    $id-prefix as xs:string)
as element(tei:w)* {
    let $texts :=
        normalize-space($text)
        => tokenize(" ")
    (: since we tokenize the text node we have to consider the position within
    the tokenized sequence for the ID creation :)
    for $iii in 1 to count($texts) return
        if (normalize-space($texts[$iii]) = "") then
            ()
        else
            element{QName("http://www.tei-c.org/ns/1.0", "w")} {
                attribute xml:id {$id-prefix || "_" || tokenize:generate-id($text) || "_w_" || $iii},
                attribute type {"token"},
                $texts[$iii]
            }
};

declare function tokenize:make-id-from-idno($TEI as element(tei:TEI))
as xs:string {
    let $idno := $TEI//tei:sourceDesc//tei:msIdentifier/tei:idno
    let $normalized :=
        replace($idno, "[\+,\.]", "")
        => replace("[\(\)=\[\]\\]", " ")
        => normalize-space()
        => replace(" |/", "_")
    return
        if ($normalized = "*") then
            (: Mlahso is the Neo-Aramaic oral record which does not have an idno. :)
            "t_Mlahso"
        else if (starts-with($normalized, "Fragment")) then
            "t_Sogdian"
        else
            (: in some cases the idno doesn't start with a letter but a digit.
            to get a uniform ID we prepend a prefix for all tokens. :)
            "t_" || $normalized
};

(:~
 : Creates a unique and stable ID for an element or attribute node.
 : Not considering the document node.
 :)
declare function tokenize:generate-id($node as node())
as xs:string? {
    "N" || (generate-id($node) => substring-after('N'))
};
