Merge branch 'remove-backports' into 'main'

Remove backporting to CI/CD templates

Closes #43

See merge request components/opentofu!71

.gitlab-ci.yml

@@ -28,8 +28,6 @@ include:
           - tests/integration.gitlab-ci.yml
           - tests/integration-tests/*.yml
           - tests/iac/**.tf
-          - backports/*.gitlab-ci.yml
-          - backports/OpenTofu/*.gitlab-ciyml
       - if: $CI_COMMIT_TAG
   # FIXME: we cannot make this work for all use cases because of the following:
   # - cannot pass parallel.matrix to the component, thus we need to extend it
@@ -122,7 +120,6 @@ gitlab-opentofu-image:build:
         - src/**/*
         - templates/**/*
         - tests/**/*
-        - backports/**/*
 
 check-readme:
   stage: test
@@ -142,23 +139,6 @@ check-readme:
         - .gitlab/README.md.template
         - templates/**/*
 
-check-backports:
-  stage: test
-  needs: []
-  image: alpine:latest
-  before_script:
-    - apk add coreutils make git sed yq diffutils patch
-  script:
-    - make backports
-    - git diff --exit-code
-  rules:
-    - if: $CI_COMMIT_TAG
-    - changes:
-        - Makefile
-        - .gitlab-ci.yml
-        - backports/**/*
-        - templates/**/*
-
 shellcheck:
   stage: test
   needs: []

CONTRIBUTING.md

@@ -20,16 +20,6 @@ All of the above definitions have to match each other.
 We currently need to change it in multiple places, because there is not a good way to share information
 from the templates and the components pipeline defintion - at least in the features we'd like to use them.
 
-## Backports
-
-The OpenTofu CI/CD component needs to be backported as OpenTofu CI/CD template,
-because components are not yet properly supported in self-managed instances.
-That is, they are not bundled and it's not possible to use a component across instances.
-
-The OpenTofu CI/CD job and pipeline templates can be generated using `make backports`.
-The output is generated into the `backports` folder.
-Please contribute those files only upon full manual inspection to the canonical GitLab repository.
-
 ## Releasing
 
 Use the `make release` command with the `VERSION` argument set to the

Makefile

-all: docs backports
+all: docs
 
 .PHONY: docs
 docs:
@@ -16,41 +16,6 @@ docs:
 	tail -n+2 readme1 >> README.md
 	rm -f readme0 readme1 readme_inputs.md
 
-BACKPORTS_DIR := backports
-BACKPORTS_BASE_DIR := $(BACKPORTS_DIR)/OpenTofu
-BACKPORTS_BASE_FILE := $(BACKPORTS_BASE_DIR)/Base.latest.gitlab-ci.yml
-
-.PHONY: backports
-
-# NOTE: this make target requires GNU sed and not the mac OS sed.
-#       Install it with `brew install gnu-sed` and follow the instructions in `brew info gnu-sed` to
-#       make it the standard `sed` binary (if you wish) or temporarily alias sed=gsed
-backports:
-	@echo "Generating $(BACKPORTS_BASE_FILE) ..."
-	@mkdir -p $(BACKPORTS_BASE_DIR)
-	@cp $(BACKPORTS_DIR)/.Base.latest.gitlab-ci.yml $(BACKPORTS_BASE_FILE)
-	@sed '1,/^---$$/d' templates/fmt.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:fmt/' | sed -e 's/$$\[\[ inputs.stage \]\]/validate/' | sed -e 's/$$\[\[ inputs.allow_failure \]\]/true/' >> $(BACKPORTS_BASE_FILE)
-	@sed '1,/^---$$/d' templates/validate.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:validate/' | sed -e 's/$$\[\[ inputs.stage \]\]/validate/' >> $(BACKPORTS_BASE_FILE)
-	@sed '1,/^---$$/d' templates/plan.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:plan/' | sed -e 's/$$\[\[ inputs.stage \]\]/build/' >> $(BACKPORTS_BASE_FILE)
-	@sed '1,/^---$$/d' templates/apply.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:apply/' | sed -e 's/$$\[\[ inputs.stage \]\]/deploy/' | sed -e 's/"$$\[\[ inputs.auto_apply \]\]"/$$_TF_AUTO_APPLY/' >> $(BACKPORTS_BASE_FILE)
-	@sed '1,/^---$$/d' templates/destroy.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:destroy/' | sed -e 's/$$\[\[ inputs.stage \]\]/cleanup/' | sed -e 's/"$$\[\[ inputs.auto_destroy \]\]"/$$_TF_AUTO_DESTROY/' | sed -e 's/$$\[\[ inputs.create_destroy_job \]\]/$$TF_CREATE_DESTROY_JOB/' >> $(BACKPORTS_BASE_FILE)
-	@sed '1,/^---$$/d' templates/delete-state.yml | sed -e 's/$$\[\[ inputs.as \]\]/.opentofu:delete-state/' | sed -e 's/$$\[\[ inputs.stage \]\]/cleanup/' | sed -e 's/$$\[\[ inputs.create_delete_state_job \]\]/$$TF_CREATE_DELETE_STATE_JOB/' >> $(BACKPORTS_BASE_FILE)
-
-	@# Common inputs
-	@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.image_registry_base \]\]/$$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/'
-	@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.image_name \]\]/gitlab-opentofu/'
-	@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.version \]\]/$$GITLAB_OPENTOFU_VERSION/'
-	@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.opentofu_version \]\]/$$OPENTOFU_VERSION/'
-	@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.root_dir \]\]/$$TF_ROOT/'
-	@sed -i $(BACKPORTS_BASE_FILE) -e 's/$$\[\[ inputs.state_name \]\]/$$TF_STATE_NAME/'
-	@sed -i $(BACKPORTS_BASE_FILE) -e '/TF_STATE_NAME: $$TF_STATE_NAME/d'
-	@sed -i $(BACKPORTS_BASE_FILE) -e '/TF_ROOT: $$TF_ROOT/d'
-
-	@# Remove empty variable blocks - yq doesn't preserve white spaces, thus the patching ...
-	@yq e 'del(.*.variables | select(length==0))' $(BACKPORTS_BASE_FILE) | diff -Bw $(BACKPORTS_BASE_FILE) - | patch $(BACKPORTS_BASE_FILE) -
-
-	@echo "Generated $(BACKPORTS_BASE_FILE)"
-
 .PHONY: release
 release:
 	@[ -n "$(VERSION)" ] || (echo "Please provide a VERSION argument for this release" && false)

backports/.Base.latest.gitlab-ci.yml

-# This template is a port of the OpenTofu CI/CD component at
-# https://gitlab.com/components/opentofu
-# It is generated with the `make backports` command from that project.
-#
-# Please make sure to use the component (https://gitlab.com/components/opentofu)
-# when your project is hosted on GitLab.com
-# or when you are willing to mirror the component project into your self-managed
-# instance and use it from there.
-#
-# Attention: This template will be removed in favor of the OpenTofu CI/CD component following components
-#            are available for self-managed instances.
-#
-# This specific template is located at:
-# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenTofu/Base.latest.gitlab-ci.yml
-
-variables:
-  # OpenTofu CI/CD component version, see https://gitlab.com/components/opentofu/-/releases
-  GITLAB_OPENTOFU_VERSION: "latest"
-  # Compatible OpenTofu version, see https://gitlab.com/components/opentofu/-/releases
-  OPENTOFU_VERSION: "1.6.0"
-  # Job Image with `gitlab-tofu`
-  GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE: $CI_REGISTRY/components/opentofu
-  # The relative path to the root directory of the OpenTofu project
-  TF_ROOT: ${CI_PROJECT_DIR}
-  # The name of the state file used by the GitLab Managed Terraform state backend
-  TF_STATE_NAME: default
-
-opentofu:use-component-instead-of-template:
-  stage: .pre
-  needs: []
-  allow_failure: true
-  rules:
-    - if: '$CI_SERVER_HOST == "gitlab.com"'
-  image: alpine:3.19
-  script:
-    - |
-      echo "You are using the OpenTofu CI/CD template on GitLab.com, which is not recommended."
-      echo "This template is available for self-managed users only until CI/CD components are "
-      echo "available. See https://gitlab.com/gitlab-org/gitlab/-/issues/415638"
-      echo " "
-      echo "You should use the OpenTofu CI/CD component instead."
-      echo "To include the CI/CD component with a default configuration:"
-      echo " "
-      echo "include:"
-      echo "  - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/full-pipeline@<VERSION>"
-      echo "    inputs:"
-      echo "      version: <VERSION>"
-      echo "      opentofu_version: 1.6.0"
-      echo ""
-      echo "stages: [validate, build, deploy, cleanup]"
-      echo " "
-      echo "You can read about more about the OpenTofu CI/CD component here:"
-      echo "https://gitlab.com/components/opentofu"
-    - 'false'

backports/OpenTofu.latest.gitlab-ci.yml

-# This template is a port of the OpenTofu CI/CD component at
-# https://gitlab.com/components/opentofu
-# It is generated with the `make backports` command from that project.
-#
-# Please make sure to use the component when your project is hosted on GitLab.com
-# or when you are willing to mirror the component project into your self-managed
-# instance and use it from there.
-#
-# Attention: This template will be removed in favor of the OpenTofu CI/CD component as soon as components
-#            are available for self-managed instances.
-#
-# This specific template is located at:
-# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenTofu.latest.gitlab-ci.yml
-
-include:
-  - template: OpenTofu/Base.latest.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/OpenTofu/Base.latest.gitlab-ci.yml
-
-stages: [validate, build, deploy]
-
-fmt:
-  extends: .opentofu:fmt
-
-validate:
-  extends: .opentofu:validate
-
-plan:
-  extends: .opentofu:plan
-
-apply:
-  extends: .opentofu:apply

backports/OpenTofu/Base.latest.gitlab-ci.yml

-# This template is a port of the OpenTofu CI/CD component at
-# https://gitlab.com/components/opentofu
-# It is generated with the `make backports` command from that project.
-#
-# Please make sure to use the component (https://gitlab.com/components/opentofu)
-# when your project is hosted on GitLab.com
-# or when you are willing to mirror the component project into your self-managed
-# instance and use it from there.
-#
-# Attention: This template will be removed in favor of the OpenTofu CI/CD component following components
-#            are available for self-managed instances.
-#
-# This specific template is located at:
-# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenTofu/Base.latest.gitlab-ci.yml
-
-variables:
-  # OpenTofu CI/CD component version, see https://gitlab.com/components/opentofu/-/releases
-  GITLAB_OPENTOFU_VERSION: "latest"
-  # Compatible OpenTofu version, see https://gitlab.com/components/opentofu/-/releases
-  OPENTOFU_VERSION: "1.6.0"
-  # Job Image with `gitlab-tofu`
-  GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE: $CI_REGISTRY/components/opentofu
-  # The relative path to the root directory of the OpenTofu project
-  TF_ROOT: ${CI_PROJECT_DIR}
-  # The name of the state file used by the GitLab Managed Terraform state backend
-  TF_STATE_NAME: default
-
-opentofu:use-component-instead-of-template:
-  stage: .pre
-  needs: []
-  allow_failure: true
-  rules:
-    - if: '$CI_SERVER_HOST == "gitlab.com"'
-  image: alpine:3.19
-  script:
-    - |
-      echo "You are using the OpenTofu CI/CD template on GitLab.com, which is not recommended."
-      echo "This template is available for self-managed users only until CI/CD components are "
-      echo "available. See https://gitlab.com/gitlab-org/gitlab/-/issues/415638"
-      echo " "
-      echo "You should use the OpenTofu CI/CD component instead."
-      echo "To include the CI/CD component with a default configuration:"
-      echo " "
-      echo "include:"
-      echo "  - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/full-pipeline@<VERSION>"
-      echo "    inputs:"
-      echo "      version: <VERSION>"
-      echo "      opentofu_version: 1.6.0"
-      echo ""
-      echo "stages: [validate, build, deploy, cleanup]"
-      echo " "
-      echo "You can read about more about the OpenTofu CI/CD component here:"
-      echo "https://gitlab.com/components/opentofu"
-    - 'false'
-
-'.opentofu:fmt':
-  stage: validate
-  needs: []
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $CI_OPEN_MERGE_REQUESTS  # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
-      when: never
-    - if: $CI_COMMIT_BRANCH        # If there's no open merge request, add it to a *branch* pipeline instead.
-  allow_failure: true
-  cache:
-    key: "$__CACHE_KEY_HACK"
-    paths:
-      - $TF_ROOT/.terraform/
-  variables:
-    # FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
-    __CACHE_KEY_HACK: "$TF_ROOT"
-  image:
-    name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
-  script:
-    - gitlab-tofu fmt
-
-'.opentofu:validate':
-  stage: validate
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $CI_OPEN_MERGE_REQUESTS  # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
-      when: never
-    - if: $CI_COMMIT_BRANCH        # If there's no open merge request, add it to a *branch* pipeline instead.
-  cache:
-    key: "$__CACHE_KEY_HACK"
-    paths:
-      - $TF_ROOT/.terraform/
-  variables:
-    # FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
-    __CACHE_KEY_HACK: "$TF_ROOT"
-  image:
-    name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
-  script:
-    - gitlab-tofu validate
-
-'.opentofu:plan':
-  stage: build
-  environment:
-    name: $TF_STATE_NAME
-    action: prepare
-  resource_group: $TF_STATE_NAME
-  artifacts:
-    # Terraform's cache files can include secrets which can be accidentally exposed.
-    # Please exercise caution when utilizing secrets in your Terraform infrastructure and
-    # consider limiting access to artifacts or take other security measures to protect sensitive information.
-    #
-    # The next line, which disables public access to pipeline artifacts, is not available on GitLab.com.
-    # See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic
-    public: false
-    paths:
-      - $TF_ROOT/plan.cache
-    reports:
-      terraform: $TF_ROOT/plan.json
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $CI_OPEN_MERGE_REQUESTS  # Don't add it to a *branch* pipeline if it's already in a merge request pipeline.
-      when: never
-    - if: $CI_COMMIT_BRANCH        # If there's no open merge request, add it to a *branch* pipeline instead.
-  cache:
-    key: "$__CACHE_KEY_HACK"
-    paths:
-      - $TF_ROOT/.terraform/
-  variables:
-    # FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
-    __CACHE_KEY_HACK: "$TF_ROOT"
-  image:
-    name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
-  script:
-    - gitlab-tofu plan
-    - gitlab-tofu plan-json
-
-'.opentofu:apply':
-  stage: deploy
-  environment:
-    name: $TF_STATE_NAME
-    action: start
-  resource_group: $TF_STATE_NAME
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $_TF_AUTO_APPLY == "true"'
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      when: manual
-  cache:
-    key: "$__CACHE_KEY_HACK"
-    paths:
-      - $TF_ROOT/.terraform/
-  variables:
-    # FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
-    __CACHE_KEY_HACK: "$TF_ROOT"
-  image:
-    name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
-  script:
-    - gitlab-tofu apply
-
-'.opentofu:destroy':
-  stage: cleanup
-  environment:
-    name: $TF_STATE_NAME
-    action: stop
-  resource_group: $TF_STATE_NAME
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $_TF_AUTO_DESTROY == "true"'
-    - when: manual
-  cache:
-    key: "$__CACHE_KEY_HACK"
-    paths:
-      - $TF_ROOT/.terraform/
-  variables:
-    # FIXME: work around to make slashes work in `cache:key`. see https://gitlab.com/gitlab-org/gitlab/-/issues/439898
-    __CACHE_KEY_HACK: "$TF_ROOT"
-  image:
-    name: '$GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE/gitlab-opentofu:$GITLAB_OPENTOFU_VERSION-opentofu$OPENTOFU_VERSION'
-  script:
-    - gitlab-tofu destroy
-
-'.opentofu:delete-state':
-  stage: cleanup
-  resource_group: $TF_STATE_NAME
-  image: curlimages/curl:latest
-  script:
-    - curl --request DELETE -u "gitlab-ci-token:$CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/terraform/state/$TF_STATE_NAME"
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
-    - when: manual

backports/README.md

-# OpenTofu CI/CD template backports
-
-See [CONTRIBUTING.md](/CONTRIBUTING.md#backports)

tests/integration-tests/BackportTemplates.gitlab-ci.yml

-include:
-  - local: /backports/OpenTofu/Base.latest.gitlab-ci.yml
-
-stages: [validate, test, build, deploy, cleanup]
-
-# Required to run everything immediately, instead of manually.
-
-fmt:
-  extends: .opentofu:fmt
-  rules: [{when: always}]
-
-validate:
-  extends: .opentofu:validate
-  rules: [{when: always}]
-
-plan:
-  extends: .opentofu:plan
-  rules: [{when: always}]
-
-apply:
-  extends: .opentofu:apply
-  rules: [{when: always}]
-
-destroy:
-  extends: .opentofu:destroy
-  rules: [{when: always}]
-
-delete-state:
-  extends: .opentofu:delete-state
-  needs: [destroy]
-  rules: [{when: always}]
-

tests/integration.gitlab-ci.yml

@@ -12,18 +12,3 @@ component:
       - PIPELINE_NAME: [Defaults]
       - PIPELINE_NAME: [JobTemplates]
       - PIPELINE_NAME: [TestJob]
-
-backport-templates:
-  stage: test-integration
-  variables:
-    GITLAB_OPENTOFU_IMAGE_REGISTRY_BASE: $GITLAB_OPENTOFU_IMAGE_BASE
-    GITLAB_OPENTOFU_VERSION: $CI_COMMIT_SHA
-    OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
-    TF_STATE_NAME: ci-integration-backports-$CI_PIPELINE_IID-$CI_NODE_INDEX
-    TF_ROOT: tests/iac
-  trigger:
-    include: tests/integration-tests/$PIPELINE_NAME.gitlab-ci.yml
-    strategy: depend
-  parallel:
-    matrix:
-      - PIPELINE_NAME: [BackportTemplates]