wip: test timeout for server creation

cleanup.py

@@ -10,17 +10,35 @@ def main() -> None:
     print("Delete openstack instances", flush=True)
     try:
         conn = openstack.connect()
+
         for server in conn.compute.servers(name=env.VM_NAME):
+            volumes = server.list_volumes()
             for i in range(5):
                 conn.delete_server(server.id, delete_ips=True)
                 time.sleep(5)
                 state = conn.compute.find_server(server.id, ignore_missing=True)
                 if state is None:
-                    print(f'Delete server {server.id} successful')
+                    print(f'Delete server {server.id} successful', flush=True)
                     break
+            
+            for volume in volumes:
+                start_time = time.time()
+                while time.time() - start_time < 300:  # Keep trying for up to 5 minutes
+                    try:
+                        if conn.block_storage.find_volume(volume.id):
+                            conn.block_storage.delete_volume(volume.id)
+                        print(f"Successfully deleted volume with ID {volume.id}", flush=True)
+                        return
+                    except openstack.exceptions.SDKException as e:
+                        print(f"Error deleting volume with ID {volume.id}: {e}", flush=True)
+                        print("Retrying in 10 seconds...", flush=True)
+                        time.sleep(10)
+                print(f"Unable to delete volume with ID {volume.id} after 5 minutes", flush=True)
+
         if os.path.exists(env.PRIVATE_KEY_PATH):
             os.remove(env.PRIVATE_KEY_PATH)
         conn.delete_keypair(env.KEY_PAIR_NAME)
+  
         conn.close()
     except Exception as e:
         traceback.print_exc()

env.py

@@ -14,3 +14,4 @@ FLOATING_IP_NETWORK = os.getenv("FLOATING_IP_NETWORK") or "public"
 SSH_IP_VERSION = os.getenv("SSH_IP_VERSION") or "4"
 BUILD_FAILURE_EXIT_CODE = os.getenv("BUILD_FAILURE_EXIT_CODE")
 SYSTEM_FAILURE_EXIT_CODE = os.getenv("SYSTEM_FAILURE_EXIT_CODE")
+SERVER_CREATION_TIMEOUT = os.getenv("SERVER_CREATION_TIMEOUT") or "180"

prepare.py

@@ -23,21 +23,25 @@ def provision_server(
     image = conn.compute.find_image(env.BUILDER_IMAGE)
     flavor = conn.compute.find_flavor(env.FLAVOR)
     network = conn.network.find_network(env.NETWORK)
-    server = conn.create_server(
-        name=env.VM_NAME,
-        flavor=flavor.id,
-        image=image.id,
-        boot_from_volume=True,
-        terminate_volume=True,
-        wait=True,
-        timeout=300,
-        volume_size=env.VOLUME_SIZE,
-        key_name=env.KEY_PAIR_NAME,
-        security_groups=[group for group in env.SECURITY_GROUPS.split()],
-        network=network.id
-    )
+    try:
+        server = conn.create_server(
+            name=env.VM_NAME,
+            flavor=flavor.id,
+            image=image.id,
+            boot_from_volume=True,
+            terminate_volume=True,
+            volume_size=env.VOLUME_SIZE,
+            key_name=env.KEY_PAIR_NAME,
+            security_groups=[group for group in env.SECURITY_GROUPS.split()],
+            network=network.id
+        )
+    except openstack.exceptions.SDKException as e:
+        print(e, flush=True)
+    except Exception as e:
+        print(e, flush=True)
 
-    server = conn.wait_for_server(server, timeout=180)
+    print(f'Waiting for server to come up...')
+    server = conn.wait_for_server(server, timeout=int(env.SERVER_CREATION_TIMEOUT))
     
     return server
 
@@ -104,7 +108,6 @@ def generate_rsa_keypair():
     with open(env.PRIVATE_KEY_PATH, 'w') as content_file:
         content_file.write(pem.decode('utf-8'))
     public_key_str = public_key.decode('utf-8')
-    print(f'Public Key: {public_key_str}')
     return public_key_str
 
 
@@ -118,6 +121,7 @@ def main() -> None:
         conn = openstack.connect()
         print(f"Provisioning an instance {env.VM_NAME}", flush=True)
         public_key = generate_rsa_keypair()
+        print(f'Public Key: {public_key}', flush=True)
         server = provision_server(conn, public_key)
         ip = get_server_ip(conn, server)
         print(f"Instance {env.VM_NAME} is running on address {ip}", flush=True)