Listen to the keypress on the dialog, call 'close' method to close the dialog. Please look at this code:
var dlg = $('#dlg');
dlg.window('window').attr('tabindex',1).focus().bind('keyup', function(e){
if (e.keyCode == 27){ // ESC
dlg.window('close');
}
})