someone helped me with this and came up with this wich seems to do a better job except it doesnt stack all detected cloned nicks and send in 1 or 2 go

Code

ON !^*:JOIN:#test: {
  if (!$nick($chan,$me,@&~%)) && (o !isin $usermode) { return }

  var %hash_1 = JCLONES_ $+ $chan
  var %hash_2 = JCLONESNICKS_ $+ $chan
  var %nick_5chars = $left($nick,5)

  hinc -mu3 %hash_1 %nick_5chars 1
  hadd -mu3 %hash_2 $nick $mask($fulladdress,4)
  var %nick_total = $hget(%hash_1,%nick_5chars)

  if (%nick_total > 1) {

    var %t = $hget(%hash_2,0).item
    if (%t) {
      var %i = 1
      while (%i <= %t) {
        var %n = $hget(%hash_2,%i).item
        var %h = $gettok($hget(%hash_2,%n),1,32)

        if (%n) && (%h) && (* $+ %nick_5chars $+ * iswm %n) {
          var %total_nicks = $addtok(%total_nicks,%n,44)
          var %total_hosts = $addtok(%total_hosts,%h,32)
        }
        inc %i
      }
      if (!$Timer(#)) { .raw -q MODE $chan +RMb $+(*,%nick_5chars,?*!*@*) | .Timer $+ $chan 1 5 MODE $chan -MR  }
      if (%total_hosts) { putmode $chan $+(+,$str(b,$numtok(%total_hosts,32))) %total_hosts }
      if (%total_nicks) { .raw -q KICK $chan %total_nicks IDenticaL NICK(s) }
    }
    if ($hget(%hash_1)) { hfree $v1 }
    if ($hget(%hash_2)) { hfree $v1 }
  }
}


and it doesnt seem get all identical nicks only in pairs of 2 so if 3 will join with same first 5 chars in nick only first 2 will get detected same goes for 5 7 9 11 13 15 17 and so on
the last one never gets detected and the nicks arent stacked as ircd allows stacked kicks and bans