diff --git a/src/xfpm-button.c b/src/xfpm-button.c
index cdc2e56ef18384aa561b002eb1627c5d6a06fa81..c70f979df336419f1263185e1c40f6e7611eb29b 100644
--- a/src/xfpm-button.c
+++ b/src/xfpm-button.c
@@ -147,6 +147,7 @@ xfpm_button_keysym_to_code_mask (XfpmButton *button,
         }
         retval = TRUE;
       }
+      XkbFreeKeyboard (xkbmap, XkbAllClientInfoMask, TRUE);
     }
     else
     {