You need to evaluate the $+(%,rl.,$comchan($nick,%i),$address($nick,2)) var before you can compare its value
Code:
    while ($comchan($nick,%i)) {
      var %c = $ifmatch
      if ($me isop %c) {
        inc -u3600 $+(%,rl.,%c,$wildsite)
        var %a = [color:blue]$([/color]$+(%,rl.,%c,$wildsite)[color:blue],2)[/color]
        if %a == 1 { .notice $nick Please don't ctcp flood this is your first warning. }
        elseif %a == 2 {
          ban -u60 %c $nick 11
          notice $nick You have been placed on notice ban for ctcp flood 1 minute ban.
        }
        else ban -ku3600 %c $nick 2 You were warned! CTCP flood 60 minute ban.
      }
      inc %i
    }