diff --git a/lib/rpc_backend b/lib/rpc_backend
index 3177e88..5479db3 100644
--- a/lib/rpc_backend
+++ b/lib/rpc_backend
@@ -97,6 +97,8 @@
 
             break
         done
+        # NOTE(frickler): Remove the default guest user
+        sudo rabbitmqctl delete_user guest || true
     fi
 }