Initial commit

.pre-commit-config.yaml

+repos:
+  - repo: https://github.com/asottile/reorder_python_imports
+    rev: v2.5.0
+    hooks:
+      - id: reorder-python-imports
+        language_version: python3
+  - repo: https://github.com/psf/black
+    rev: 21.6b0
+    hooks:
+      - id: black
+        args: [--safe, --quiet, --line-length, "100"]
+        language_version: python3
+        require_serial: true
+  - repo: https://gitlab.com/pycqa/flake8.git
+    rev: 3.9.2
+    hooks:
+      - id: flake8
+        language_version: python3
+        args:
+          - --max-line-length=100
+          - --ignore=W503,E203
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: trailing-whitespace
+        language_version: python3
+      - id: end-of-file-fixer
+        language_version: python3
+      - id: debug-statements
+        language_version: python3
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.19.3
+    hooks:
+      - id: pyupgrade
+        language_version: python3

Containerfile

+ARG GITLAB_RUNNER_VERSION=master
+
+FROM registry.access.redhat.com/ubi8:8.4 AS builder
+
+ARG GITLAB_RUNNER_VERSION
+
+ENV GITLAB_REPO=https://gitlab.com/gitlab-org/gitlab-runner.git \
+    PATH=$PATH:/root/go/bin/
+
+RUN dnf install -y git-core make go ncurses && \
+    git clone --depth=1 --branch=${GITLAB_RUNNER_VERSION} ${GITLAB_REPO} && \
+    cd gitlab-runner && \
+    make runner-bin-host && \
+    chmod a+x out/binaries/gitlab-runner && \
+    out/binaries/gitlab-runner --version
+
+FROM registry.access.redhat.com/ubi8:8.4
+
+ARG GITLAB_RUNNER_VERSION
+
+COPY --from=builder /gitlab-runner/out/binaries/gitlab-runner /usr/bin
+
+ENV HOME=/home/gitlab-runner \
+    VENV=/openstack_driver_venv
+
+ENV PATH="$VENV/bin:$PATH"
+
+LABEL maintainer="Dmitry Misharov <misharov@redhat.com>" \
+      version="$GITLAB_RUNNER_VERSION" \
+      io.openshift.tags="gitlab,ci,runner" \
+      name="openstack-gitlab-runner" \
+      io.k8s.display-name="GitLab runner" \
+      summary="GitLab runner" \
+      description="A GitLab runner image with openstack custom executor." \
+      io.k8s.description="A GitLab runner image with openstack custom executor."
+
+WORKDIR $HOME
+
+COPY cleanup.py config.py prepare.py run.py requirements.txt start.sh .
+
+RUN dnf install -y --nodocs python38-pip git-core && \
+    python3.8 -m venv $VENV && \
+    pip install wheel && \
+    pip install -r requirements.txt && \
+    dnf remove -y git-core && \
+    dnf clean all -y
+
+RUN chgrp -R 0 $HOME && \
+    chmod +x cleanup.py config.py prepare.py run.py start.sh && \
+    chmod -R g=u $HOME
+
+USER 1001
+
+CMD ["./start.sh"]

cleanup.py

+#!/usr/bin/env python
+import openstack
+
+import config
+
+
+def main() -> None:
+    print(f"Deleting instance {config.VM_NAME}")
+    conn = openstack.connect()
+    server = conn.compute.find_server(config.VM_NAME)
+    conn.compute.delete_server(server)
+
+
+if __name__ == "__main__":
+    main()

config.py

+import os
+
+VM_NAME = f"gitlab-builder-{os.getenv('CUSTOM_ENV_CI_RUNNER_ID')}-project-{os.getenv('CUSTOM_ENV_CI_PROJECT_ID')}-concurrent-{os.getenv('CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID')}-job-{os.getenv('CUSTOM_ENV_CI_JOB_ID')}"  # noqa
+
+FLAVOR = os.getenv("CUSTOM_ENV_FLAVOR") or os.getenv("FLAVOR")
+BUILDER_IMAGE = os.getenv("CUSTOM_ENV_BUILDER_IMAGE") or os.getenv("BUILDER_IMAGE")
+NETWORK = os.getenv("CUSTOM_ENV_NETWORK") or os.getenv("NETWORK")
+KEY_PAIR_NAME = os.getenv("CUSTOM_ENV_KEY_PAIR_NAME") or os.getenv("KEY_PAIR_NAME")
+SECURITY_GROUP = os.getenv("CUSTOM_ENV_SECURITY_GROUP") or os.getenv("SECURITY_GROUP")
+USERNAME = os.getenv("CUSTOM_ENV_USERNAME") or os.getenv("USERNAME")
+PRIVATE_KEY_PATH = os.getenv("PRIVATE_KEY_PATH")

prepare.py

+#!/usr/bin/env python
+import openstack
+import paramiko
+
+import config
+
+
+def provision_server(conn: openstack.connection.Connection) -> openstack.compute.v2.server.Server:
+    image = conn.compute.find_image(config.BUILDER_IMAGE)
+    flavor = conn.compute.find_flavor(config.FLAVOR)
+    network = conn.network.find_network(config.NETWORK)
+    server = conn.compute.create_server(
+        name=config.VM_NAME,
+        flavor_id=flavor.id,
+        image_id=image.id,
+        key_name=config.KEY_PAIR_NAME,
+        security_groups=[{"name": config.SECURITY_GROUP}],
+        networks=[{"uuid": network.id}],
+    )
+    return conn.compute.wait_for_server(server, wait=600)
+
+
+def get_server_ip(
+    conn: openstack.connection.Connection, server: openstack.compute.v2.server.Server
+) -> str:
+    return list(conn.compute.server_ips(server))[0].address
+
+
+def check_ssh(ip: str) -> None:
+    ssh_client = paramiko.client.SSHClient()
+    pkey = paramiko.rsakey.RSASHA256Key.from_private_key_file(config.PRIVATE_KEY_PATH)
+    ssh_client.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
+    ssh_client.connect(
+        hostname=ip,
+        username=config.USERNAME,
+        pkey=pkey,
+        look_for_keys=False,
+        allow_agent=False,
+        timeout=60,
+    )
+    ssh_client.close()
+
+
+def main() -> None:
+    print("Connecting to Openstack")
+    conn = openstack.connect()
+    print(f"Provisioning an instance '{config.VM_NAME}'")
+    server = provision_server(conn)
+    ip = get_server_ip(conn, server)
+    print(f"Instance {config.VM_NAME} is running on address {ip}")
+    conn.close()
+    print("Checking SSH connection")
+    check_ssh(ip)
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

+appdirs==1.4.4
+bcrypt==3.2.0
+certifi==2021.5.30
+cffi==1.14.5
+chardet==4.0.0
+cryptography==3.4.7
+decorator==5.0.9
+dogpile.cache==1.1.3
+idna==2.10
+iso8601==0.1.14
+jmespath==0.10.0
+jsonpatch==1.32
+jsonpointer==2.1
+keystoneauth1==4.3.1
+munch==2.5.0
+netifaces==0.11.0
+openstacksdk==0.57.0
+os-service-types==1.7.0
+paramiko @ git+https://github.com/kkovaacs/paramiko.git@d43ed76a44c451b2b9031883a1a7e55de59fdf79 # adds support of modern RSA keys
+pbr==5.6.0
+pycparser==2.20
+PyNaCl==1.4.0
+PyYAML==5.4.1
+requests==2.25.1
+requestsexceptions==1.4.0
+six==1.16.0
+stevedore==3.3.0
+urllib3==1.26.5

run.py

+#!/usr/bin/env python
+import stat
+import sys
+from pathlib import Path
+
+import openstack
+import paramiko
+
+import config
+
+
+def get_server_ip(conn: openstack.connection.Connection) -> str:
+    server = conn.compute.find_server(config.VM_NAME)
+    return list(conn.compute.server_ips(server))[0].address
+
+
+def execute_script_on_server(
+    ssh: paramiko.client.SSHClient, sftp: paramiko.sftp_client.SFTPClient, script_path: str
+) -> None:
+    path = Path(script_path)
+    sftpattrs = sftp.put(script_path, path.name)
+    sftp.chmod(path.name, sftpattrs.st_mode | stat.S_IEXEC)
+    _, stdout, _ = ssh.exec_command(f"./{path.name}")
+    print(*stdout.readlines())
+    sftp.remove(path.name)
+
+
+def get_ssh_client(ip: str) -> paramiko.client.SSHClient:
+    ssh_client = paramiko.client.SSHClient()
+    pkey = paramiko.rsakey.RSASHA256Key.from_private_key_file(config.PRIVATE_KEY_PATH)
+    ssh_client.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
+    ssh_client.connect(
+        hostname=ip,
+        username=config.USERNAME,
+        pkey=pkey,
+        look_for_keys=False,
+        allow_agent=False,
+        timeout=60,
+    )
+    return ssh_client
+
+
+def main() -> None:
+    conn = openstack.connect()
+    ip = get_server_ip(conn)
+    ssh_client = get_ssh_client(ip)
+    sftp_client = ssh_client.open_sftp()
+    print(f"Executing {sys.argv[2]}")
+    execute_script_on_server(ssh_client, sftp_client, sys.argv[1])
+    sftp_client.close()
+    ssh_client.close()
+
+
+if __name__ == "__main__":
+    main()

start.sh

+#!/bin/bash
+
+trap cleanup &>/proc/1/fd/1 1 2 3 6
+
+cleanup() {
+    gitlab-runner unregister --all-runners
+    sleep 5
+}
+
+if [[ "$TLS_CA_CERT" ]]; then
+    mkdir -p /home/gitlab-runner/.gitlab-runner/certs/
+    echo "$TLS_CA_CERT" > /home/gitlab-runner/.gitlab-runner/certs/$(echo "$CI_SERVER_URL" | cut -d'/' -f3 | cut -d':' -f1).crt
+fi
+
+gitlab-runner register --non-interactive \
+                       --executor=custom \
+                       --builds-dir="$HOME"/builds \
+                       --cache-dir="$HOME"/cache \
+                       --custom-prepare-exec="$HOME"/prepare.py \
+                       --custom-run-exec="$HOME"/run.py \
+                       --custom-cleanup-exec="$HOME"/cleanup.py
+
+gitlab-runner run