Merge branch 'opa-policy-enforcement' into 'main'

Implement new `policy-enforcement` template

See merge request components/opentofu!295

.gitlab/README.md.template

@@ -328,6 +328,51 @@ merge request overview page. For this, the component supports the
 This feature requires a `GITLAB_TOFU_TOKEN` CI/CD variable to be present.
 It must contain an access token with `api` scope and at least `Reporter` role.
 
+### Policy Enforcement with Open Policy Agent (OPA) (experimental)
+
+This component supports a new and experimental feature that allows you to enforce
+policies on your OpenTofu plan with the Open Policy Agent (OPA).
+Currently, this feature is supported with the `policy-enforcement` template.
+The policies are evaluated with the decision given with the `opa_decision` input (default is `tofu`).
+The policy bundle must be configured in the config provided (default `opa-config.yml`).
+When the script at `.gitlab/ci/opa-post-processing` is present, it's run after `opa` and the
+OPA result JSON is provided in `stdin`. This can be used to analyse the OPA result
+and fail the job if necessary.
+
+#### Example: use versioned policy bundle from GitLab OCI registry
+
+To use a versioned policy bundle from the GitLab OCI registry you can configure
+your `opa-config.yml` like this:
+
+```yml
+services:
+  gitlab-registry:
+    url: https://registry.gitlab.com
+    type: oci
+    credentials:
+      bearer:
+        scheme: "Basic"
+        token: "gitlab-ci-token:$CI_JOB_TOKEN"
+
+bundles:
+  policies:
+    service: gitlab-registry
+    resource: registry.gitlab.com/<group>/<project>/<oci-policy-bundle-name>:<version>
+    persist: true
+```
+
+The component automatically runs `envsubst` on your provided `opa-config.yml` so
+that tokens can be integrated in the configuration in a secure way.
+
+The policy bundle can be published to the GitLab OCI registry using `opa` and `oras`:
+
+```shell
+oras login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
+opa build tests/iac-policy/policy
+echo '{}' > config.json
+oras push "${CI_REGISTRY_IMAGE}/iac-policy:${CI_COMMIT_SHA}" --config config.json:application/vnd.oci.image.config.v1+json bundle.tar.gz:application/vnd.oci.image.layer.v1.tar+gzip
+```
+
 ### Access to Terraform Module Registry
 
 Similar to automatically configuring the [GitLab-managed Terraform state backend]

Dockerfile.alpine

@@ -26,7 +26,8 @@ RUN apk add --no-cache \
   jq \
   openssh-client \
   cosign \
-  unzip
+  unzip \
+  envsubst
 
 # Install OpenTofu using the installer script in standalone mode
 # see https://opentofu.org/docs/intro/install/standalone

Dockerfile.debian

@@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y \
   openssh-client \
   unzip \
   idn2 \
+  gettext-base \
   && rm -rf /var/lib/apt/lists/*
 
 # NOTE: cosign is not yet available in the debian apt sources

README.md

@@ -330,6 +330,51 @@ merge request overview page. For this, the component supports the
 This feature requires a `GITLAB_TOFU_TOKEN` CI/CD variable to be present.
 It must contain an access token with `api` scope and at least `Reporter` role.
 
+### Policy Enforcement with Open Policy Agent (OPA) (experimental)
+
+This component supports a new and experimental feature that allows you to enforce
+policies on your OpenTofu plan with the Open Policy Agent (OPA).
+Currently, this feature is supported with the `policy-enforcement` template.
+The policies are evaluated with the decision given with the `opa_decision` input (default is `tofu`).
+The policy bundle must be configured in the config provided (default `opa-config.yml`).
+When the script at `.gitlab/ci/opa-post-processing` is present, it's run after `opa` and the
+OPA result JSON is provided in `stdin`. This can be used to analyse the OPA result
+and fail the job if necessary.
+
+#### Example: use versioned policy bundle from GitLab OCI registry
+
+To use a versioned policy bundle from the GitLab OCI registry you can configure
+your `opa-config.yml` like this:
+
+```yml
+services:
+  gitlab-registry:
+    url: https://registry.gitlab.com
+    type: oci
+    credentials:
+      bearer:
+        scheme: "Basic"
+        token: "gitlab-ci-token:$CI_JOB_TOKEN"
+
+bundles:
+  policies:
+    service: gitlab-registry
+    resource: registry.gitlab.com/<group>/<project>/<oci-policy-bundle-name>:<version>
+    persist: true
+```
+
+The component automatically runs `envsubst` on your provided `opa-config.yml` so
+that tokens can be integrated in the configuration in a secure way.
+
+The policy bundle can be published to the GitLab OCI registry using `opa` and `oras`:
+
+```shell
+oras login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
+opa build tests/iac-policy/policy
+echo '{}' > config.json
+oras push "${CI_REGISTRY_IMAGE}/iac-policy:${CI_COMMIT_SHA}" --config config.json:application/vnd.oci.image.config.v1+json bundle.tar.gz:application/vnd.oci.image.layer.v1.tar+gzip
+```
+
 ### Access to Terraform Module Registry
 
 Similar to automatically configuring the [GitLab-managed Terraform state backend]

templates/policy-enforcement.yml

+spec:
+  inputs:
+    # Job and Stage name
+    as:
+      default: 'policy-enforcement'
+      description: 'Defines the name of this job.'
+    stage:
+      default: 'build'
+      description: 'Defines the stage that this job will belong to.'
+
+    # Versions
+    # This version is only required, because we cannot access the context of the component,
+    # see https://gitlab.com/gitlab-org/gitlab/-/issues/438275
+    version:
+      default: '2.3.0'
+      description: 'Version of this component. Has to be the same as the one in the component include entry.'
+
+    base_os:
+      default: 'alpine'
+      options:
+        - 'alpine'
+        - 'debian'
+        - '$GITLAB_OPENTOFU_BASE_IMAGE_OS'
+      description: 'Base OS of GitLab OpenTofu image.'
+
+    opentofu_version:
+      default: '1.9.1'
+      options:
+        - '1.9.1'
+        - '1.9.0'
+        - '1.8.9'
+        - '1.8.8'
+        - '1.7.8'
+        - '1.7.7'
+        - '1.6.3'
+        - '$OPENTOFU_VERSION'
+      description: 'OpenTofu version that should be used.'
+
+    # Images
+    image_registry_base:
+      default: '$CI_TEMPLATE_REGISTRY_HOST/components/opentofu'
+      description: 'Host URI to the job images. Will be combined with `image_name` to construct the actual image URI.'
+    # FIXME: not yet possible because of https://gitlab.com/gitlab-org/gitlab/-/issues/438722
+    # gitlab_opentofu_image:
+    #   # FIXME: This should reference the component tag that is used.
+    #   #        Currently, blocked by https://gitlab.com/gitlab-org/gitlab/-/issues/438275
+    #   # default: '$CI_TEMPLATE_REGISTRY_HOST/components/opentofu/gitlab-opentofu:$[[ inputs.opentofu_version ]]'
+    #   default: '$CI_TEMPLATE_REGISTRY_HOST/components/opentofu/gitlab-opentofu:$[[ inputs.version ]]-opentofu$[[ inputs.opentofu_version ]]'
+    #   description: 'Tag of the gitlab-opentofu image.'
+
+    image_name:
+      default: 'gitlab-opentofu'
+      description: 'Image name for the job images. Hosted under `image_registry_base`.'
+
+    image_digest:
+      default: ''
+      # FIXME: we cannot use regex yet because of a bug that rejects
+      # empty strings from ever being checked against the regex.
+      # see https://gitlab.com/gitlab-org/gitlab/-/issues/477707
+      # regex: '^(@sha256:[a-z0-9]{64})?$'
+      description: 'Image digest of the image you want to use. The format must be `@<image_digest>`, e.g. `@sha256:abc..`, see regex of this input. Please consult the release page at https://gitlab.com/components/opentofu/-/releases to obtain the image digests.'
+
+    # Configuration
+    root_dir:
+      default: ${CI_PROJECT_DIR}
+      description: 'Root directory for the OpenTofu project.'
+    plan_name:
+      default: 'plan'
+      description: 'The name of the plan cache and plan json file.'
+    var_file:
+      default: ''
+      type: string
+      description: 'Path to a variables files relative to root_dir.'
+    rules:
+      # FIXME: eventually, we'll want to define `null` as the default,
+      # but this is NOT support yet, see
+      # https://gitlab.com/gitlab-org/gitlab/-/issues/440468
+      default: [{when: on_success}]
+      type: array
+      description: 'Defines the `rules` of the job.'
+    artifacts_access:
+      default: 'none'
+      description: 'Access level for the OPA result JSON artifact. See https://docs.gitlab.com/ee/ci/yaml/#artifactsaccess for possible values.'
+    auto_encryption:
+      default: false
+      type: boolean
+      description: 'Whether to enable automatic state and plan encryption.'
+    auto_encryption_passphrase:
+      default: ''
+      type: string
+      description: 'Defines the passphrase to auto encrypt the state and plan. Only used if `auto_encryption` is `true`.'
+    auto_encryption_enable_migration_from_unencrypted:
+      default: false
+      type: boolean
+      description: 'Whether to setup automatic state and plan encryption for currently unencrypted state. This is only temporarily useful when migrating from an unencrypted state.'
+    opa_version:
+      default: '1.4.2'
+      type: string
+      description: 'Open Policy Agent (OPA) version to download and run.'
+    opa_decision:
+      default: 'tofu'
+      type: string
+      description: 'OPA exec decision to evaluate.'
+    opa_config:
+      default: 'opa-config.yml'
+      type: string
+      description: 'Path to the OPA configuration file.'
+    opa_post_processing_exec:
+      default: '.gitlab/ci/opa-post-processing'
+      type: string
+      description: 'Path to an executable to run after OPA. The OPA result JSON is provided in stdin.'
+    plan_job_name:
+      default: 'plan'
+      type: string
+      description: 'Name of the plan job to fetch the plan file from.'
+
+---
+
+
+'$[[ inputs.as ]]':
+  stage: $[[ inputs.stage ]]
+  needs:
+    - $[[ inputs.plan_job_name ]]
+  rules: $[[ inputs.rules ]]
+  variables:
+    GITLAB_TOFU_ROOT_DIR: $[[ inputs.root_dir ]]
+    GITLAB_TOFU_PLAN_NAME: $[[ inputs.plan_name ]]
+    GITLAB_TOFU_VAR_FILE: '$[[ inputs.var_file ]]'
+    GITLAB_TOFU_AUTO_ENCRYPTION: '$[[ inputs.auto_encryption ]]'
+    GITLAB_TOFU_AUTO_ENCRYPTION_PASSPHRASE: '$[[ inputs.auto_encryption_passphrase ]]'
+    GITLAB_TOFU_AUTO_ENCRYPTION_ENABLE_MIGRATION_FROM_UNENCRYPTED: '$[[ inputs.auto_encryption_enable_migration_from_unencrypted ]]'
+  image:
+    name: '$[[ inputs.image_registry_base ]]/$[[ inputs.image_name ]]:$[[ inputs.version ]]-opentofu$[[ inputs.opentofu_version ]]-$[[ inputs.base_os ]]$[[ inputs.image_digest ]]'
+  cache:
+    key: "gitlab-tofu-opa-$[[ inputs.opa_version ]]-${CI_RUNNER_EXECUTABLE_ARCH}"
+    paths:
+      - ./.gitlab-tofu/bin/tmp
+      - $GITLAB_TOFU_ROOT_DIR/.terraform/
+  artifacts:
+    access: '$[[ inputs.artifacts_access ]]'
+    paths:
+      - opa-result.json
+  before_script:
+    - |
+      if [ -f "./.gitlab-tofu/bin/tmp/opa" ]; then
+        echo "OPA version '$[[ inputs.opa_version ]]' for '${CI_RUNNER_EXECUTABLE_ARCH}' found in cache, reusing it."
+      else
+        arch="$(echo "${CI_RUNNER_EXECUTABLE_ARCH}" | cut -d '/' -f2)"
+        echo "Downloading OPA version '$[[ inputs.opa_version ]]' for arch '$arch' ..."
+        mkdir -p ./.gitlab-tofu/bin/tmp
+        wget "https://github.com/open-policy-agent/opa/releases/download/v$[[ inputs.opa_version ]]/opa_linux_${arch}_static" -O ./opa
+        chmod +x ./opa
+        mv ./opa ./.gitlab-tofu/bin/tmp
+      fi
+    - ./.gitlab-tofu/bin/tmp/opa version
+    - echo "Running envsubst on '$[[ inputs.opa_config ]]' ..."
+    - envsubst < "$[[ inputs.opa_config ]]" > /tmp/opa-config.env.yml
+  script:
+    - gitlab-tofu init
+    - gitlab-tofu show -json "$[[ inputs.plan_name ]].cache" > plan.json
+    - ./.gitlab-tofu/bin/tmp/opa exec -c /tmp/opa-config.env.yml --decision "$[[ inputs.opa_decision ]]" plan.json | tee opa-result.json
+    - |
+      if [ -f "$[[ inputs.opa_post_processing_exec ]]" ]; then
+        echo "Found post processing script at '$[[ inputs.opa_post_processing_exec ]]', running it with OPA result JSON streaming to stdin ..."
+        "$[[ inputs.opa_post_processing_exec ]]" < opa-result.json
+      else
+        echo "Post processing script at '$[[ inputs.opa_post_processing_exec ]]' not found, skipping it ..."
+      fi

tests/iac-policy/main.tf

+resource "random_pet" "this" {
+  length = 2
+}
+
+# resource "random_password" "this" {
+#   length = 2
+# }

tests/iac-policy/policy/tofu.rego

+package tofu
+
+import input as tfplan
+
+# Only consider these types
+resource_types := {"random_pet", "random_password"}
+
+#########
+# Policy
+#########
+
+# Authorization holds if no password resource is touched
+default allow := false
+
+allow if {
+	not touches_password
+}
+
+# Whether there is any change to a random_password resource
+touches_password if {
+	all_resources := resources.random_password
+	count(all_resources) > 0
+}
+
+####################
+# Terraform Library
+####################
+
+# list of all resources of a given type
+resources[resource_type] := all_resources if {
+	some resource_type, _ in resource_types
+
+	all_resources := [name |
+		some name in tfplan.resource_changes
+		name.type == resource_type
+	]
+}

tests/integration-tests/PolicyEnforcement.gitlab-ci.yml

+include:
+  - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/plan@$CI_COMMIT_SHA
+    inputs:
+      image_registry_base: $GITLAB_OPENTOFU_IMAGE_BASE
+      version: $CI_COMMIT_SHA
+      base_os: $GITLAB_OPENTOFU_BASE_IMAGE_OS
+      opentofu_version: $OPENTOFU_VERSION
+      root_dir: $TEST_GITLAB_TOFU_ROOT_DIR
+      state_name: $TEST_GITLAB_TOFU_STATE_NAME
+      rules: [{when: on_success}]
+  - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/policy-enforcement@$CI_COMMIT_SHA
+    inputs:
+      image_registry_base: $GITLAB_OPENTOFU_IMAGE_BASE
+      version: $CI_COMMIT_SHA
+      base_os: $GITLAB_OPENTOFU_BASE_IMAGE_OS
+      opentofu_version: $OPENTOFU_VERSION
+      root_dir: $TEST_GITLAB_TOFU_ROOT_DIR
+      rules: [{when: on_success}]
+
+stages: [setup, build]
+
+policy-enforcement:
+  needs:
+    - plan        # default
+    - policy:push # we need that explicitly because of the opa-config.yml artifact.
+
+policy:push:
+  stage: setup
+  needs: []
+  rules: [{when: always}]
+  image: alpine:latest
+  artifacts:
+    paths:
+      - opa-config.yml
+  before_script:
+    - wget "https://github.com/open-policy-agent/opa/releases/download/v1.4.2/opa_linux_amd64_static" -O ./opa
+    - chmod +x ./opa
+    - mv ./opa /usr/bin/
+    - opa version
+    - wget "https://github.com/oras-project/oras/releases/download/v1.2.3/oras_1.2.3_linux_amd64.tar.gz" -O ./oras.tar.gz
+    - tar xfvz ./oras.tar.gz
+    - chmod +x ./oras
+    - mv ./oras /usr/bin/
+    - oras version
+  script:
+    - oras login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
+    - opa build tests/iac-policy/policy
+    - echo '{}' > config.json
+    - oras push "${CI_REGISTRY_IMAGE}/iac-policy:${CI_COMMIT_SHA}" --config config.json:application/vnd.oci.image.config.v1+json bundle.tar.gz:application/vnd.oci.image.layer.v1.tar+gzip
+    # Write OPA config to download bundles
+    - |
+      cat <<EOF > opa-config.yml
+      services:
+        gitlab-registry:
+          url: https://${CI_REGISTRY}
+          type: oci
+          credentials:
+            bearer:
+              scheme: "Bearer"
+              token: "\$CI_JOB_TOKEN"
+
+      bundles:
+        policies:
+          service: gitlab-registry
+          resource: ${CI_REGISTRY_IMAGE}/iac-policy:${CI_COMMIT_SHA}
+          persist: true
+      EOF

tests/integration.gitlab-ci.yml

@@ -211,3 +211,21 @@ upload-artifacts:
           - alpine
           - debian
 
+
+policy-enforcement:
+  stage: test-integration
+  variables:
+    OPENTOFU_VERSION: $LATEST_OPENTOFU_VERSION
+    TEST_GITLAB_TOFU_STATE_NAME: ci-integration-$CI_JOB_NAME_SLUG-$CI_PIPELINE_IID-$CI_NODE_INDEX
+    TEST_GITLAB_TOFU_ROOT_DIR: tests/iac-policy
+  trigger:
+    include: tests/integration-tests/$PIPELINE_NAME.gitlab-ci.yml
+    strategy: depend
+  parallel:
+    matrix:
+      - PIPELINE_NAME:
+          - PolicyEnforcement
+        GITLAB_OPENTOFU_BASE_IMAGE_OS:
+          - alpine
+          - debian
+