diff --git a/cleanup.py b/cleanup.py
index 39617a3858b09aca94a254251f455a1b55615e95..c3ee77dd74a825b53878a7b9ea6e8a93751f7f94 100644
--- a/cleanup.py
+++ b/cleanup.py
@@ -1,14 +1,17 @@
 #!/usr/bin/env python
 import openstack
-
 import env
+import os
 
 
 def main() -> None:
     conn = openstack.connect()
     for server in conn.compute.servers(name=env.VM_NAME):
         conn.compute.delete_server(server)
+    if os.path.exists(env.PRIVATE_KEY_PATH):
+        os.remove(env.PRIVATE_KEY_PATH)
     conn.delete_keypair(env.KEY_PAIR_NAME)
 
+
 if __name__ == "__main__":
     main()
diff --git a/env.py b/env.py
index b8a7baa5ac743bcb1d9832e0143a1289cc90ae53..f7ea9dbf4386b67be89947f9f74fe2a6995e26a7 100644
--- a/env.py
+++ b/env.py
@@ -1,14 +1,13 @@
 import os
 
 VM_NAME = f"gitlab-builder-{os.getenv('CUSTOM_ENV_CI_RUNNER_ID')}-project-{os.getenv('CUSTOM_ENV_CI_PROJECT_ID')}-concurrent-{os.getenv('CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID')}-job-{os.getenv('CUSTOM_ENV_CI_JOB_ID')}"  # noqa
-
 FLAVOR = os.getenv("CUSTOM_ENV_FLAVOR") or os.getenv("FLAVOR")
 BUILDER_IMAGE = os.getenv("CUSTOM_ENV_BUILDER_IMAGE") or os.getenv("BUILDER_IMAGE")
 NETWORK = os.getenv("CUSTOM_ENV_NETWORK") or os.getenv("NETWORK")
 KEY_PAIR_NAME = f'key-{VM_NAME}'
 SECURITY_GROUPS = os.getenv("CUSTOM_ENV_SECURITY_GROUPS") or os.getenv("SECURITY_GROUPS")
 USERNAME = os.getenv("CUSTOM_ENV_USERNAME") or os.getenv("USERNAME")
-PRIVATE_KEY_PATH = f"{os.getenv('HOME')}/priv_key"
+PRIVATE_KEY_PATH = f"{os.getenv('HOME')}/priv_key-{VM_NAME}"
 SSH_TIMEOUT = os.getenv("CUSTOM_ENV_SSH_TIMEOUT") or os.getenv("SSH_TIMEOUT") or "20"
 FLOATING_IP_NETWORK = os.getenv("FLOATING_IP_NETWORK") or "public"
 SSH_IP_VERSION = os.getenv("SSH_IP_VERSION") or "4"