/*
{ -------------------------------------------------------------------------------
$tok(<name>, <text>, [, parm, parm, ...])[.cs]
Returns the value of a call to $<name>tok[cs](<text>, [, parm1, parm2, ...])
but acknowledges null tokens both in <text> and appropriate parms.
Use the .cs property to call the case sensitive version of the token identifier.
Examples:
$tok(num, a..b..c, 46) = 5
$tok(rem, a..b..c, , 2, 46) = a..b.c
$tok(put, a..c, b, 2, 46) = a.b.c
$tok(rep, a..c, c, , 46) = a..
$tok(find, .a.A, A, 46).cs = 3
Note that if you plan to use this for sorting, the randomly selected placeholder,
chosen from the 256-55295 code point range will result in null tokens being placed
after characters in the typically used 1-255 range.
} -------------------------------------------------------------------------------
*/
alias tok {
while ($chr($rand(256, 55295)) isin $2) /
var %marker = $v1 | ; random placeholder
var %C = $iif($1 == sort, $3, $eval($ $+ $0, 2))
if (%C !isnum) return
var %i = 3, %parms, %delim = $+(\Q, $chr(%C), \E), %cs
if ($prop == cs) && ($istok(add find is match rem rep sort wild, $1, 32)) %cs = cs
if ($0 > 2) && ($3 == $null) && ($istok(add find ins is put rem rep wild, $1, 32)) {
%i = 4
%parms = ,%marker
}
while (%i <= $0) {
if (%i == 4) && ($1 == rep) && ($4 == $null) {
%parms = %parms ,%marker
}
else {
%parms = %parms ,$ $+ %i
}
inc %i
}
returnex $remove($eval($+($, $1, tok, %cs, ( $eval( $&
$regsubex(__tokn,$2, /(?<=^| %delim )(?=\z| %delim )/gx, %marker) $&
, 0) %parms )), 2), %marker)
}
/*
{ -------------------------------------------------------------------------------
$tokcs(<name>, <text>, [, parm, parm, ...])
Syntactic sugar for $tok().cs.
For example:
$tokcs(rep, A..a., a, , 46) = $tok(rep, A..a., a, , 46).cs = A...
} -------------------------------------------------------------------------------
*/
alias tokcs {
var %i = 1, %parms
while (%i <= $0) {
%parms = %parms ,$ $+ %i
inc %i
}
returnex $tok( [ $mid(%parms, 2) ] ).cs
}