Change cp -p to mv.
diff --git a/stack.sh b/stack.sh
index 9cb7341..cfec6ca 100755
--- a/stack.sh
+++ b/stack.sh
@@ -139,8 +139,7 @@
     sed -e "s,%USER%,$USER,g" -i $TEMPFILE
     chmod 0440 $TEMPFILE
     sudo chown root:root $TEMPFILE
-    sudo cp -p $TEMPFILE /etc/sudoers.d/stack_sh_nova
-    sudo rm $TEMPFILE
+    sudo mv $TEMPFILE /etc/sudoers.d/stack_sh_nova
 fi
 
 # Set the destination directories for openstack projects