Merge branch 'unit-testing-bats' into 'main'

Use bats instead of job based tests

See merge request components/opentofu!76

.gitlab-ci.yml

@@ -15,6 +15,7 @@ include:
           - Dockerfile
           - .gitlab-ci.yml
           - tests/unit.gitlab-ci.yml
+          - tests/unit/*
       - if: $CI_COMMIT_TAG
   - local: tests/integration.gitlab-ci.yml
     rules:

tests/unit.gitlab-ci.yml

 variables:
-  TEST_PROJECT_DIR: 'tests/iac'
+  TEST_PROJECT_DIR: $CI_PROJECT_DIR/tests/iac
 
-.gitlab-tofu-test-base:
+.unit-test-base:
   image: "$GITLAB_OPENTOFU_IMAGE_NAME"
-  variables:
-    TF_STATE_NAME: ci-unit-$CI_JOB_ID
-  cache:
-    key: "$OPENTOFU_VERSION-$CI_COMMIT_REF_SLUG"
-    paths:
-      - $TEST_PROJECT_DIR/.terraform/
   before_script:
-    - gitlab-tofu version
+    # Install dependencies
+    - apk add bats parallel
+    - mkdir -p /tmp/bats-libs
+    - git clone --depth 1 --branch v0.3.0 https://github.com/bats-core/bats-support.git /tmp/bats-libs/bats-support
+    - git clone --depth 1 --branch v2.1.0 https://github.com/bats-core/bats-assert.git /tmp/bats-libs/bats-assert
+    - export BATS_LIBS=/tmp/bats-libs
+    # List versions
+    - bats --version
     - jq --version
-  after_script:
-    - curl --request DELETE -u "gitlab-ci-token:$CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/terraform/state/$TF_STATE_NAME"
-
-.gitlab-tofu-test:
-  extends:
-    - .gitlab-tofu-test-base
-  before_script:
-    - !reference [.gitlab-tofu-test-base, before_script]
-    - cd $TEST_PROJECT_DIR
-
-.test-gitlab-tofu-root:
-  extends:
-    - .gitlab-tofu-test-base
-  variables:
-    TF_ROOT: $TEST_PROJECT_DIR
-
-gitlab-tofu-init:
-  extends:
-    - .gitlab-tofu-test
-    - .opentofu-versions
-  stage: test
-  script:
-    - export DEBUG_OUTPUT=true
-    - gitlab-tofu init
-
-gitlab-tofu-init-with-args:
-  extends:
-    - .gitlab-tofu-test
-    - .opentofu-versions
-  stage: test
-  script:
-    - export DEBUG_OUTPUT=true
-    - gitlab-tofu init -get=true -no-color
-
-gitlab-tofu-init-with-flags:
-  extends:
-    - .gitlab-tofu-test
-    - .opentofu-versions
-  stage: test
-  script:
-    - export DEBUG_OUTPUT=true
-    - export TF_INIT_FLAGS="-get=true -no-color"
-    - gitlab-tofu init
-
-gitlab-tofu-init-with-flags-and-args:
-  extends:
-    - .gitlab-tofu-test
-    - .opentofu-versions
-  stage: test
-  script:
-    - export DEBUG_OUTPUT=true
-    - export TF_INIT_FLAGS="-get=true"
-    - gitlab-tofu init -no-color
-
-gitlab-tofu-init-tf-root:
-  extends:
-    - .test-gitlab-tofu-root
-    - .opentofu-versions
-  stage: test
-  script:
-    - export DEBUG_OUTPUT=true
-    - gitlab-tofu init
-
-gitlab-tofu-init-tf-root-with-cd:
-  extends:
-    - .test-gitlab-tofu-root
-    - .opentofu-versions
-  stage: test
-  script:
-    - cd $TEST_PROJECT_DIR
-    - export DEBUG_OUTPUT=true
-    - gitlab-tofu init
-
-gitlab-tofu-init-tf-root-with-args:
-  extends:
-    - .test-gitlab-tofu-root
-    - .opentofu-versions
-  stage: test
-  script:
-    - export DEBUG_OUTPUT=true
-    - gitlab-tofu init -get=true -no-color
-
-gitlab-tofu-init-tf-root-with-flags:
-  extends:
-    - .test-gitlab-tofu-root
-    - .opentofu-versions
-  stage: test
-  script:
-    - export DEBUG_OUTPUT=true
-    - export TF_INIT_FLAGS="-get=true -no-color"
-    - gitlab-tofu init
-
-gitlab-tofu-init-tf-root-with-flags-and-args:
-  extends:
-    - .test-gitlab-tofu-root
-    - .opentofu-versions
-  stage: test
-  script:
-    - export DEBUG_OUTPUT=true
-    - export TF_INIT_FLAGS="-get=true"
-    - gitlab-tofu init -no-color
-
-gitlab-tofu-init-without-reconfigure:
-  extends:
-    - .test-gitlab-tofu-root
-    - .opentofu-versions
-  stage: test
-  script:
-    - gitlab-tofu init
-    - |
-      cat <<EOF > $TF_ROOT/backend_override.tf
-      terraform {
-        backend "local" {}
-      }
-      EOF
-    - export TF_INIT_NO_RECONFIGURE=true
-    - FAILED=false
-    - gitlab-tofu init -no-color >/tmp/output.txt 2>&1 || FAILED=true
-    - cat /tmp/output.txt
-    - test $FAILED = true
-    - 'grep "Error: Backend configuration changed" /tmp/output.txt'
-
-gitlab-tofu-init-with-reconfigure:
-  extends:
-    - .test-gitlab-tofu-root
-    - .opentofu-versions
-  stage: test
-  script:
-    - gitlab-tofu init
-    - |
-      cat <<EOF > $TF_ROOT/backend_override.tf
-      terraform {
-        backend "local" {}
-      }
-      EOF
-    - gitlab-tofu init
-
-gitlab-tofu-init-with-prepared-registry-token:
-  extends:
-    - .gitlab-tofu-test
-  stage: test
-  variables:
-    OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
-  script:
-    - apk add --update $PKG
-    - |
-      cat <<'EOF' > test.sh
-      set -x
-      # NOTE: as part of the test fixture, we need to overwrite the CI_SERVER_HOST,
-      # so that this test also properly works on GitLab self-managed.
-      export CI_SERVER_HOST=gitlab.example.com
-      export TF_TOKEN_gitlab_example_com=mysecrettoken
-      . $(which gitlab-tofu)
-      terraform_authenticate_private_registry
-      test "$TF_TOKEN_gitlab_example_com" = "mysecrettoken"
-      EOF
-    - $SHELL test.sh
-  parallel:
-    matrix:
-      - SHELL: "bash"
-        PKG: "bash"
-      - SHELL: "zsh"
-        PKG: "zsh"
-      - SHELL: "ksh"
-        PKG: "loksh"
-
-gitlab-tofu-init-without-prepared-registry-token:
-  extends:
-    - .gitlab-tofu-test
-  stage: test
-  variables:
-    OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
-  script:
-    - apk add --update $PKG
-    - |
-      cat <<'EOF' > test.sh
-      set -x
-      # NOTE: as part of the test fixture, we need to overwrite the CI_SERVER_HOST,
-      # so that this test also properly works on GitLab self-managed.
-      export CI_SERVER_HOST=gitlab.example.com
-      . $(which gitlab-tofu)
-      terraform_authenticate_private_registry
-      test -n "$TF_TOKEN_gitlab_example_com"
-      EOF
-    - $SHELL test.sh
-  parallel:
-    matrix:
-      - SHELL: "bash"
-        PKG: "bash"
-      - SHELL: "zsh"
-        PKG: "zsh"
-      - SHELL: "ksh"
-        PKG: "loksh"
-
-gitlab-tofu-fmt:
-  extends:
-    - .gitlab-tofu-test
-    - .opentofu-versions
-  stage: test
-  script:
-    - gitlab-tofu fmt
-
-gitlab-tofu-validate:
-  extends:
-    - .gitlab-tofu-test
-    - .opentofu-versions
-  stage: test
-  script:
-    - gitlab-tofu validate
-
-gitlab-tofu-plan:
-  extends:
-    - .gitlab-tofu-test
-    - .opentofu-versions
-  stage: test
-  variables:
-    TF_PLAN_CACHE: $OPENTOFU_VERSION-plan.cache
-  script:
-    - gitlab-tofu plan
-    - if [[ ! -f "$OPENTOFU_VERSION-plan.cache" ]]; then echo "expected to find a plan.cache file"; exit 1; fi
-    - gitlab-tofu plan-json
-    - if [[ ! -f "plan.json" ]]; then echo "expected to find a plan.json file"; exit 1; fi
+    - gitlab-tofu version
   artifacts:
+    when: always
     paths:
-      - "$TEST_PROJECT_DIR/*-plan.cache"
+      - report.xml
+    reports:
+      junit: report.xml
 
-gitlab-tofu-apply:
+unit-test:gitlab-tofu:
   extends:
-    - .gitlab-tofu-test
+    - .unit-test-base
     - .opentofu-versions
-  stage: test
-  variables:
-    TF_PLAN_CACHE: $OPENTOFU_VERSION-plan.cache
-  before_script:
-    - !reference [.gitlab-tofu-test, before_script]
-    - gitlab-tofu plan
   script:
-    - gitlab-tofu apply
-
-gitlab-tofu-destroy:
-  extends:
-    - .gitlab-tofu-test
-    - .opentofu-versions
-  stage: test
-  before_script:
-    - !reference [.gitlab-tofu-test, before_script]
-    - gitlab-tofu plan
-    - gitlab-tofu apply
-  script:
-    - gitlab-tofu destroy
-
-gitlab-tofu-source-script:
-  extends:
-    - .gitlab-tofu-test
-  stage: test
-  variables:
-    OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
-  before_script:
-    - !reference [.gitlab-tofu-test-base, before_script]
-    - apk add --update $PKG
-  script:
-    - |
-      cat <<'EOF' > test.sh
-      set -x
-      test -z "$TF_GITLAB_SOURCED"
-      . $(which gitlab-tofu)
-      test $TF_GITLAB_SOURCED
-      EOF
-    - |
-      mkdir /usr/local/sbin
-      cat <<'EOF' > /usr/local/sbin/terraform
-      #/!usr/bin/env sh -e
-      echo "Called Terraform, but shouldn't have!!"
-      false
-      EOF
-      chmod +x /usr/local/sbin/terraform
-    - $SHELL test.sh
-  parallel:
-    matrix:
-      - SHELL: "bash"
-        PKG: "bash"
-      - SHELL: "zsh"
-        PKG: "zsh"
-      - SHELL: "ksh"
-        PKG: "loksh"
-
-gitlab-tofu-without-implicit-init:
-  extends:
-    - .gitlab-tofu-test
-  stage: test
-  cache:
-  variables:
-    OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
-    STATE_NAME: $CI_JOB_NAME
-  script:
-    - export TF_IMPLICIT_INIT=false
-    - FAILED=false
-    - gitlab-tofu $CMD -no-color >/tmp/output.txt 2>&1 || FAILED=true
-    - cat /tmp/output.txt
-    - test $FAILED = true
-    - 'grep "$ERROR" /tmp/output.txt'
-  parallel:
-    matrix:
-      - CMD: apply
-        ERROR: 'Error: Failed to load "plan.cache" as a plan'
-      - CMD: destroy
-        ERROR: 'Error: Backend initialization required, please run "tofu init"'
-      - CMD: plan
-        ERROR: 'Error: Backend initialization required, please run "tofu init"'
-      - CMD: validate
-        ERROR: 'This module is not yet installed. Run "tofu init" to install all modules'
-
-gitlab-tofu-no-wrapper:
-  extends:
-    - .gitlab-tofu-test
-  stage: test
-  cache:
-  variables:
-    OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
-    STATE_NAME: $CI_JOB_NAME
-  script:
-    # NOTE: running `gitlab-tofu apply` wouldn't fail
-    #       because of the implicit `terraform init`.
-    - FAILED=false
-    - gitlab-tofu -- apply -no-color >/tmp/output.txt 2>&1 || FAILED=true
-    - cat /tmp/output.txt
-    - test $FAILED = true
-    - 'grep "Error: Backend initialization required, please run \"tofu init\"" /tmp/output.txt'
-
-gitlab-tofu-state-name-auto-urlencode:
-  extends:
-    - .gitlab-tofu-test
-  stage: test
-  variables:
-    OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
-  script:
-    - apk add --update $PKG
-    - |
-      cat <<'EOF' > test.sh
-      set -x
-      export TF_STATE_NAME=production/europe
-      . $(which gitlab-tofu)
-      test "$TF_STATE_NAME" = "production%2Feurope"
-      EOF
-    - $SHELL test.sh
-  parallel:
-    matrix:
-      - SHELL: "bash"
-        PKG: "bash"
-      - SHELL: "zsh"
-        PKG: "zsh"
-      - SHELL: "ksh"
-        PKG: "loksh"
+    - bats --jobs 8 --report-formatter junit --filter-tags '!source' tests/unit/gitlab-tofu.bats
 
-gitlab-tofu-state-name-auto-urlencode-ff-disabled:
+unit-test:gitlab-tofu:source:
   extends:
-    - .gitlab-tofu-test
-  stage: test
+    - .unit-test-base
   variables:
     OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
   script:
-    - apk add --update $PKG
-    - |
-      cat <<'EOF' > test.sh
-      set -x
-      export TF_FF_AUTO_URLENCODE_STATE_NAME=false
-      export TF_STATE_NAME=production/europe
-      . $(which gitlab-tofu)
-      test "$TF_STATE_NAME" = "production/europe"
-      EOF
-    - $SHELL test.sh
+    - apk add "$PKG"
+    - bats --jobs 8 --report-formatter junit --filter-tags 'source' tests/unit/gitlab-tofu.bats
   parallel:
     matrix:
       - SHELL: "bash"

tests/unit/gitlab-tofu.bats

+#!/usr/bin/env bats
+
+bats_require_minimum_version 1.5.0
+
+# NOTE: BATS_LIBS must be set to the directory where bats-assert is installed.
+load "$BATS_LIBS/bats-support/load"
+load "$BATS_LIBS/bats-assert/load"
+
+setup() {
+  export DEBUG_OUTPUT=true
+
+  # Change cwd to test specific directory
+  cd "$BATS_TEST_TMPDIR"
+
+  # Move TF test project directory to a temporary location
+  cp -r "$TEST_PROJECT_DIR" "$BATS_TEST_TMPDIR"
+
+  # Set TF root directory to temporary location
+  export TF_ROOT="$BATS_TEST_TMPDIR/$(basename "$TEST_PROJECT_DIR")"
+
+  # Set state so that each test has its own
+  export TF_STATE_NAME="ci-unit-$CI_JOB_ID-$BATS_SUITE_TEST_NUMBER"
+}
+
+teardown() {
+  curl --request DELETE -u "gitlab-ci-token:$CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/terraform/state/$TF_STATE_NAME"
+}
+
+@test "gitlab-tofu init" {
+  gitlab-tofu init
+}
+
+@test "gitlab-tofu init with args" {
+  gitlab-tofu init -get=true -no-color
+}
+
+@test "gitlab-tofu init with environment variable flags" {
+  export TF_INIT_FLAGS="-get=true -no-color"
+  gitlab-tofu init
+}
+
+@test "gitlab-tofu init with args and environment variable flags" {
+  export TF_INIT_FLAGS="-get=true"
+  gitlab-tofu init -no-color
+}
+
+@test "gitlab-tofu init within TF_ROOT set" {
+  cd "$TF_ROOT"
+  unset "$TF_ROOT"
+  gitlab-tofu init -no-color
+}
+
+@test "gitlab-tofu init without reconfigure" {
+  gitlab-tofu init
+
+  cat <<EOF > $TF_ROOT/backend_override.tf
+  terraform {
+    backend "local" {}
+  }
+EOF
+
+  export TF_INIT_NO_RECONFIGURE="true"
+  run ! gitlab-tofu init -no-color
+  assert_output --partial "Error: Backend configuration changed"
+}
+
+@test "gitlab-tofu init with reconfigure" {
+  cat <<EOF > $TF_ROOT/backend_override.tf
+  terraform {
+    backend "local" {}
+  }
+EOF
+  gitlab-tofu init
+}
+
+@test "gitlab-tofu fmt" {
+  gitlab-tofu fmt
+}
+
+@test "gitlab-tofu validate" {
+  gitlab-tofu validate
+}
+
+@test "gitlab-tofu plan" {
+  export TF_PLAN_CACHE="test-plan.cache"
+  gitlab-tofu plan
+  if [ ! -f "$TF_ROOT/$TF_PLAN_CACHE" ]; then 
+    echo "expected to find a plan.cache file"
+    exit 1
+  fi
+
+  gitlab-tofu plan-json
+  if [ ! -f "$TF_ROOT/plan.json" ]; then 
+    echo "expected to find a plan.json file"
+    exit 1
+  fi
+}
+
+@test "gitlab-tofu apply" {
+  export TF_PLAN_CACHE="test-plan.cache"
+  gitlab-tofu plan
+  gitlab-tofu apply
+}
+
+@test "gitlab-tofu destroy" {
+  export TF_PLAN_CACHE="test-plan.cache"
+  gitlab-tofu plan
+  gitlab-tofu apply
+  gitlab-tofu destroy
+}
+
+@test "gitlab-tofu validate without implicit init" {
+  export TF_IMPLICIT_INIT=false
+ 
+  run ! gitlab-tofu validate -no-color
+  assert_output --partial 'This module is not yet installed'
+}
+
+@test "gitlab-tofu plan without implicit init" {
+  export TF_IMPLICIT_INIT=false
+ 
+  run ! gitlab-tofu plan -no-color
+  assert_output --partial 'Error: Backend initialization required'
+}
+
+@test "gitlab-tofu apply without implicit init" {
+  export TF_IMPLICIT_INIT=false
+ 
+  run ! gitlab-tofu apply -no-color
+  assert_output --partial 'Error: Failed to load '
+}
+
+@test "gitlab-tofu destroy without implicit init" {
+  export TF_IMPLICIT_INIT=false
+ 
+  run ! gitlab-tofu destroy -no-color
+  assert_output --partial 'Error: Backend initialization required'
+}
+
+@test "gitlab-tofu no wrap" {
+  # NOTE: running `gitlab-tofu apply` wouldn't fail
+  #       because of the implicit `terraform init`.
+  run gitlab-tofu -- apply -no-color
+  assert_failure
+  assert_output --partial 'Error: Backend initialization required, please run "tofu init"'
+}
+
+# bats test_tags=source
+@test "gitlab-tofu source" {
+  test -n "$SHELL"
+
+  cat <<'EOF' > test.sh
+set -x
+test -z "$TF_GITLAB_SOURCED"
+. $(which gitlab-tofu)
+test "$TF_GITLAB_SOURCED"
+EOF
+
+  $SHELL test.sh
+}
+
+# bats test_tags=source
+@test "gitlab-tofu source not calling tofu binary" {
+  test -n "$SHELL"
+
+  cat <<'EOF' > test.sh
+set -x
+test -z "$TF_GITLAB_SOURCED"
+. $(which gitlab-tofu)
+test "$TF_GITLAB_SOURCED"
+EOF
+
+  mkdir /usr/local/sbin
+  cat <<'EOF' > /usr/local/sbin/tofu
+#!/usr/bin/env sh -e
+echo "Called tofu, but shouldn't have!!"
+false
+EOF
+  chmod +x /usr/local/sbin/tofu
+
+  $SHELL test.sh
+}
+
+# bats test_tags=source
+@test "gitlab-tofu source state name auto urlencode" {
+  test -n "$SHELL"
+
+  cat <<'EOF' > test.sh
+set -x
+export TF_STATE_NAME=production/europe
+. $(which gitlab-tofu)
+test "$TF_STATE_NAME" = "production%2Feurope"
+EOF
+
+  $SHELL test.sh
+}
+
+# bats test_tags=source
+@test "gitlab-tofu source state name auto urlencode FF disabled" {
+  test -n "$SHELL"
+
+  cat <<'EOF' > test.sh
+set -x
+export TF_FF_AUTO_URLENCODE_STATE_NAME=false
+export TF_STATE_NAME=production/europe
+. $(which gitlab-tofu)
+test "$TF_STATE_NAME" = "production/europe"
+EOF
+
+  $SHELL test.sh
+}
+
+# bats test_tags=source
+@test "gitlab-tofu source init with prepared registry token" {
+  test -n "$SHELL"
+
+cat <<'EOF' > test.sh
+set -x
+# NOTE: as part of the test fixture, we need to overwrite the CI_SERVER_HOST,
+# so that this test also properly works on GitLab self-managed.
+export CI_SERVER_HOST=gitlab.example.com
+export TF_TOKEN_gitlab_example_com=mysecrettoken
+. $(which gitlab-tofu)
+terraform_authenticate_private_registry
+test "$TF_TOKEN_gitlab_example_com" = "mysecrettoken"
+EOF
+
+  $SHELL test.sh
+}
+
+# bats test_tags=source
+@test "gitlab-tofu source init without prepared registry token" {
+  test -n "$SHELL"
+
+cat <<'EOF' > test.sh
+set -x
+# NOTE: as part of the test fixture, we need to overwrite the CI_SERVER_HOST,
+# so that this test also properly works on GitLab self-managed.
+export CI_SERVER_HOST=gitlab.example.com
+export TF_TOKEN_gitlab_example_com=mysecrettoken
+. $(which gitlab-tofu)
+terraform_authenticate_private_registry
+test "$TF_TOKEN_gitlab_example_com" = "mysecrettoken"
+EOF
+
+  $SHELL test.sh
+}