ah I overlooked NOT.. my code is wrong... Here's V2 with a proposed solution

Since these are strings taken to literal integer with some internal function I'd assume: atoi(String), inside these bitwise operations, mIRC itself doesn't know what "bit-depth" it is, and I'm guessing just assumes something like uint32?

If you take the number and if it's > 0, then log2(N) + 1 is the index of the most significant bit (the leftmost bit that's on)

log2(1) + 1 = 1
log2(2) + 1 = 2
log2(4) + 1 = 3
log2(8) + 1 = 4
etc...

If the most significant bit is <= 32, do the normal operation

else: perform the operation on the number at bit-depth: MostSignificantBit

Note: mIRC does not have $log2() so to simulate this: it's $log(N) // $log(2)

In the example identifier below, it does this. I had to fix and/or/xor/not based on the most significant bit of $1 or $2 (and/or/xor use $2, not doesn't)
and/or/xor steps up to 64bit(53bit actual cause of $calc() limits) if $1 or $2 has a msb > 32 where $not() steps up to an arbitrary bit-depth

Examples:

1 in Binary (32 bits): 00000000000000000000000000000001

//var %n = 1 , %msb = $iif(!%n,0,$calc($log(%n) // $log(2) + 1)) , %not = $bitwise53(%n).not | echo -a Not: %not Binary: $base(%not,10,2,%msb)
Output:
Not: 4294967294 Binary: 11111111111111111111111111111110

4294967296 in Binary (33 bits): 100000000000000000000000000000000

//var %n = 4294967296 , %msb = $iif(!%n,0,$calc($log(%n) // $log(2) + 1)) , %not = $bitwise53(%n).not | echo -a Not: %not Binary: $base(%not,10,2,%msb)
Output:
Not: 4294967295 Binary: 011111111111111111111111111111111

Code
bitwise53 {
  var %div = 2147483648 , %V1High = $calc($1 // %div) , %V1Low = $calc($1 % %div)
  if ($prop == isbit) { 
    if ($2 <= 31) { return $isbit(%V1Low,$2) }
    else { return $isbit(%V1High,$calc($2 - 31)) }
  }
  elseif ($istok(biton bitoff,$prop,32)) {
    if ($2 <= 31) { 
      if ($prop == biton) { var %V1Low = $biton(%V1Low,$2) }
      else { var %V1Low = $bitoff(%V1Low,$2) }
    }
    else { 
      if ($prop == biton) { var %V1High = $biton(%V1High,$calc($2 - 31)) }
      else { var %V1High = $bitoff(%V1High,$calc($2 - 31)) }
    }
    return $calc(%V1High * %div + %V1Low)
  }
  elseif ($istok(and or xor not,$prop,32)) {
    var %V2High = $calc($2 // %div) , %V2Low = $calc($2 % %div)
    var %V1msb = $iif(!$1,0,$calc($log($1) // $log(2) + 1)) , %V2msb = $iif(!$2,0,$calc($log($2) // $log(2) + 1))
    if ($prop == and) { 
      if (%v1msb > 32 || %v2msb > 32) { return $calc($and(%V1High,%V2High) * %div + $and(%V1Low,%V2Low)) }
      return $and($1,$2)
    }
    if ($prop == or) { 
      if (%v1msb > 32 || %v2msb > 32) { return $calc($or(%V1High,%V2High) * %div + $or(%V1Low,%V2Low)) }
      return $or($1,$2)
    }
    if ($prop == xor) { 
      if (%v1msb > 32 || %v2msb > 32) { return $calc($xor(%V1High,%V2High) * %div + $xor(%V1Low,%V2Low)) }
      return $xor($1,$2)
    }
    if ($prop == not) {
      if (%v1msb > 32) { return $base($regsubex($base($1,10,2,%v1msb),/([0-1])/g,$iif(\1,0,1)),2,10) }
      return $not($1)
    }
  }
}