diff --git a/drivers/staging/greybus/audio_manager.c b/drivers/staging/greybus/audio_manager.c
index 619bdee46ee5df9eb79e215b0c00c3ca2f367aad..9def014cd47c7f152770dc63fc908687597316ef 100644
--- a/drivers/staging/greybus/audio_manager.c
+++ b/drivers/staging/greybus/audio_manager.c
@@ -91,6 +91,7 @@ void gb_audio_manager_remove_all(void)
 	list_for_each_entry_safe(module, next, &modules_list, list) {
 		list_del(&module->list);
 		kobject_put(&module->kobj);
+		ida_simple_remove(&module_id, module->id);
 	}
 
 	is_empty = list_empty(&modules_list);