To prevent the close-on-enter behaviour, remove the "ok" style from ID "5" ("exit" button) (just add a dialog-closing command for sclick on "5").
To prevent any triggering of the currently focused ID with the ENTER key, I added a hidden button "100" that gets focus on init and on sclick.
If your letter-IDs would be all in order (I changed "10" to "15"), you could save a lot of code with a short calculation on sclick.
Code:
alias WTFB {  dialog -m WTFB WTFB }

dialog WTFB {
  title "TEST"
  size -1 -1 163 175
  option dbu
  combo 1, 4 41 91 132, size
  button "Look Up", 2, 96 41 62 12
  text "Select the Word from the word list.", 3, 101 58 57 78
  text "Choose a word", 4, 5 31 45 8
  button "Exit", 5, 110 152 37 12
  button "A", 15, 21 8 9 9
  button "B", 16, 30 8 9 9
  button "C", 17, 39 8 9 9
  button "D", 18, 48 8 9 9
  button "E", 19, 57 8 9 9
  button "F", 20, 66 8 9 9
  button "G", 21, 75 8 9 9
  button "H", 22, 84 8 9 9
  button "I", 23, 93 8 9 9
  button "J", 24, 102 8 9 9
  button "K", 25, 111 8 9 9
  button "L", 26, 120 8 9 9
  button "M", 27, 129 8 9 9
  button "N", 28, 21 20 9 9
  button "O", 29, 30 20 9 9
  button "P", 30, 39 20 9 9
  button "Q", 31, 48 20 9 9
  button "R", 32, 57 20 9 9
  button "S", 33, 66 20 9 9
  button "T", 34, 75 20 9 9
  button "U", 35, 84 20 9 9
  button "V", 36, 93 20 9 9
  button "W", 37, 102 20 9 9
  button "XYZ", 38, 111 20 27 9
  text "Choose First Letter", 41, 5 0 60 8
  button "", 100, 100 100 1 1, hide
}

on *:dialog:WTFB:*:*:{
  if ($devent == init) { did -f $dname 100 }
  elseif ($devent == sclick) {
    did -f $dname 100
    if ($did isnum 15-37) {
      var %letter = $chr($calc($did + 50))
      set %wtb_flet %letter
      echo -a test: click on ID $did = %letter
    }
    elseif ($did == 38) {
      set %wtb_flet XYZ
      echo -a test: click on ID $did = XYZ }
  }
}