helm: add chart e2e test

Xynnn007 · Xynnn007 · commit 611b8f43b2a3 · 2026-06-24T18:38:08.000+08:00
Add Helm chart e2e targets, a local-image scenario, and kbs-client
checks so CI can deploy Trustee on kind with prebuilt materials and
validate resource access policy behavior.

Assisted-by: Cursor &lt;cursoragent@cursor.com&gt;
Signed-off-by: Xynnn007 &lt;xynnn@linux.alibaba.com&gt;
diff --git a/Makefile b/Makefile
@@ -1,7 +1,7 @@
 .PHONY: test-all test-unit test-e2e \
 	test-kbs-unit test-as-unit test-trustee-cli-unit \
 	test-kbs-e2e test-as-e2e \
-	test-kbs-vault-e2e test-kbs-docker-e2e \
+	test-kbs-vault-e2e test-kbs-docker-e2e test-helm-e2e \
 	kbs-e2e-build kbs-e2e-install-deps kbs-e2e-build-bins \
 	kbs-vault-e2e-install-deps kbs-vault-e2e-run \
 	as-e2e-install-deps as-e2e-run
@@ -95,3 +95,7 @@ as-e2e-run:
 
 test-as-e2e: as-e2e-install-deps as-e2e-run
 
+# Helm chart e2e (deploy Trustee, kbs-client, undeploy; images must already be loaded)
+test-helm-e2e:
+	$(MAKE) -C deployment/helm-chart e2e-test
+
diff --git a/deployment/helm-chart/Makefile b/deployment/helm-chart/Makefile
@@ -0,0 +1,101 @@
+# Helm chart end-to-end tests (Trustee deploy + kbs-client + undeploy).
+#
+# Prerequisites: helm, kubectl, base64, diff, and a pre-built kbs-client.
+# A Kubernetes cluster must already exist (kind recommended). Cluster creation is out of band.
+#
+# Images must already be loaded into the target Kubernetes cluster before running e2e-test.
+
+REPO_ROOT := $(abspath ../..)
+CHART_DIR := $(abspath .)
+KBS_CLIENT ?= $(REPO_ROOT)/target/release/kbs-client
+
+HELM_RELEASE ?= trustee-e2e
+NAMESPACE ?= coco-trustee-e2e
+E2E_SCENARIO ?= $(CHART_DIR)/scenarios/e2e-local-images.yaml
+KUBE_CONTEXT ?=
+HELM_TIMEOUT ?= 20m
+
+E2E_IMAGE_PREFIX ?= trustee-e2e
+E2E_IMAGE_TAG ?= e2e
+KIND_CLUSTER_NAME ?= kind
+
+E2E_KBS_IMAGE_REPOSITORY := $(E2E_IMAGE_PREFIX)/kbs-grpc-as
+E2E_AS_IMAGE_REPOSITORY := $(E2E_IMAGE_PREFIX)/coco-as-grpc
+E2E_RVPS_IMAGE_REPOSITORY := $(E2E_IMAGE_PREFIX)/rvps
+
+E2E_KBS_IMAGE := $(E2E_KBS_IMAGE_REPOSITORY):$(E2E_IMAGE_TAG)
+E2E_AS_IMAGE := $(E2E_AS_IMAGE_REPOSITORY):$(E2E_IMAGE_TAG)
+E2E_RVPS_IMAGE := $(E2E_RVPS_IMAGE_REPOSITORY):$(E2E_IMAGE_TAG)
+
+HELM_KUBE_ARGS := $(if $(KUBE_CONTEXT),--kube-context '$(KUBE_CONTEXT)',)
+HELM_E2E_IMAGE_ARGS := \
+	--set-string kbs.image.repository='$(E2E_KBS_IMAGE_REPOSITORY)' \
+	--set-string kbs.image.tag='$(E2E_IMAGE_TAG)' \
+	--set-string kbs.image.pullPolicy='Never' \
+	--set-string as.image.repository='$(E2E_AS_IMAGE_REPOSITORY)' \
+	--set-string as.image.tag='$(E2E_IMAGE_TAG)' \
+	--set-string as.image.pullPolicy='Never' \
+	--set-string rvps.image.repository='$(E2E_RVPS_IMAGE_REPOSITORY)' \
+	--set-string rvps.image.tag='$(E2E_IMAGE_TAG)' \
+	--set-string rvps.image.pullPolicy='Never'
+KUBECTL := kubectl $(if $(KUBE_CONTEXT),--context '$(KUBE_CONTEXT)',)
+
+export HELM_RELEASE NAMESPACE E2E_SCENARIO KBS_CLIENT KUBECTL KUBE_CONTEXT \
+	E2E_IMAGE_TAG E2E_IMAGE_PREFIX KIND_CLUSTER_NAME
+
+.PHONY: e2e-test deploy undeploy test-client check-tools wait-ready \
+	check-docker helm-dependency-build helm-lint load-e2e-images-into-kind
+
+check-tools:
+	@command -v helm >/dev/null || (echo "missing: helm" >&2; exit 1)
+	@command -v kubectl >/dev/null || (echo "missing: kubectl" >&2; exit 1)
+
+check-docker:
+	@command -v docker >/dev/null || (echo "missing: docker" >&2; exit 1)
+
+helm-dependency-build: check-tools
+	helm dependency build '$(CHART_DIR)'
+
+helm-lint: helm-dependency-build
+	helm lint '$(CHART_DIR)' -f '$(E2E_SCENARIO)' $(HELM_E2E_IMAGE_ARGS)
+
+load-e2e-images-into-kind: check-docker
+	@command -v kind >/dev/null || (echo "missing: kind (required to load local images)" >&2; exit 1)
+	@echo "==> loading images into kind cluster $(KIND_CLUSTER_NAME)"
+	kind load docker-image --name '$(KIND_CLUSTER_NAME)' \
+		'$(E2E_KBS_IMAGE)' '$(E2E_AS_IMAGE)' '$(E2E_RVPS_IMAGE)'
+
+deploy: helm-lint
+	helm upgrade --install '$(HELM_RELEASE)' '$(CHART_DIR)' \
+		$(HELM_KUBE_ARGS) \
+		--namespace '$(NAMESPACE)' \
+		--create-namespace \
+		-f '$(E2E_SCENARIO)' \
+		$(HELM_E2E_IMAGE_ARGS) \
+		--wait \
+		--timeout '$(HELM_TIMEOUT)'
+
+wait-ready: check-tools
+	$(KUBECTL) wait --for=condition=available deployment/$(HELM_RELEASE)-rvps -n '$(NAMESPACE)' --timeout=600s
+	$(KUBECTL) wait --for=condition=available deployment/$(HELM_RELEASE)-as -n '$(NAMESPACE)' --timeout=600s
+	$(KUBECTL) wait --for=condition=available deployment/$(HELM_RELEASE)-kbs -n '$(NAMESPACE)' --timeout=600s
+
+undeploy: check-tools
+	helm uninstall '$(HELM_RELEASE)' -n '$(NAMESPACE)' $(HELM_KUBE_ARGS) 2>/dev/null || true
+	$(KUBECTL) delete namespace '$(NAMESPACE)' --ignore-not-found --timeout=120s 2>/dev/null || true
+
+test-client: check-tools wait-ready
+	@chmod +x '$(CHART_DIR)/e2e/test.sh'
+	@'$(CHART_DIR)/e2e/test.sh'
+
+# Deploy, test, then always undeploy. Images must already be present in the cluster.
+e2e-test: check-tools
+	@status=0; \
+	if [ $$status -eq 0 ]; then \
+		$(MAKE) deploy || status=$$?; \
+	fi; \
+	if [ $$status -eq 0 ]; then \
+		$(MAKE) test-client || status=$$?; \
+	fi; \
+	$(MAKE) undeploy || true; \
+	exit $$status
diff --git a/deployment/helm-chart/README.md b/deployment/helm-chart/README.md
@@ -270,6 +270,47 @@ Default **`values.yaml`** is intentionally small. Fixed on-disk paths for **Loca
 | storageBackend.postgres.mode | string | `"internal"` | Postgres source: `internal` (Bitnami subchart) or `external` (pre-created Secret). |
 | storageBackend.type | string | `"LocalFs"` | Backend type: `LocalFs`, `LocalJson`, `Postgres`, or `Memory`. When `Postgres` (or `sessionStorageType` is `Postgres`), the chart injects `POSTGRES_URL`. Only settings for the selected type take effect. |
 
+## End-to-end test
+
+Provision a **kind** cluster out of band, preload the Trustee images into it, point `kubectl` at it, then from the repository root:
+
+```bash
+kind create cluster --name kind --wait 5m
+make -C deployment/helm-chart load-e2e-images-into-kind
+make test-helm-e2e
+```
+
+`make e2e-test` assumes **KBS / AS / RVPS** images are already loaded into the cluster and deploys with [scenarios/e2e-local-images.yaml](./scenarios/e2e-local-images.yaml). The Makefile injects image repositories, tags, and `pullPolicy: Never` at install time so local runs and CI can use different preloaded image names without changing the scenario file.
+
+- **Local preloaded images**: `trustee-e2e/*:e2e`, `pullPolicy: Never` (not GHCR `:latest`)
+- **CI images**: reuses `docker-e2e-images-linux-amd64` from `workflow-call-build-docker-e2e-materials.yml` as `ghcr.io/confidential-containers/staged-images/*:latest`
+- **Storage**: bundled Postgres KV backend (`storageBackend.type: Postgres`)
+- **KBS sessions**: `sessionStorageType: Memory`
+
+Steps:
+
+1. **`helm-dependency-build`**
+2. **`helm-lint`**
+3. **`deploy`**
+4. **`test-client`** — [e2e/test.sh](./e2e/test.sh)
+5. **`undeploy`** — always attempted, even after failures
+
+Debug individually:
+
+```bash
+make -C deployment/helm-chart load-e2e-images-into-kind
+make -C deployment/helm-chart helm-lint
+make -C deployment/helm-chart deploy
+make -C deployment/helm-chart test-client
+make -C deployment/helm-chart undeploy
+```
+
+Optional variables: `E2E_IMAGE_PREFIX`, `E2E_IMAGE_TAG`, `KBS_CLIENT`.
+
+CI (`test-e2e-kbs.yml`) reuses the Docker Compose e2e image build workflow artifacts: it loads the pre-built Trustee images into kind and uses the pre-built `kbs-client` binary before running Helm e2e.
+
+`make test-client` alone does not build images or undeploy.
+
 ## Development notes
 
 1. Do not change the files under [files](./files/) unless you are updating the corresponding Helm template logic. Service config is rendered from `*.template` files; Postgres KV init SQL lives in `files/postgres-initkv.sql`.
diff --git a/deployment/helm-chart/README.md.gotmpl b/deployment/helm-chart/README.md.gotmpl
@@ -159,6 +159,47 @@ Default **`values.yaml`** is intentionally small. Fixed on-disk paths for **Loca
 
 {{ template "chart.valuesSection" . }}
 
+## End-to-end test
+
+Provision a **kind** cluster out of band, preload the Trustee images into it, point `kubectl` at it, then from the repository root:
+
+```bash
+kind create cluster --name kind --wait 5m
+make -C deployment/helm-chart load-e2e-images-into-kind
+make test-helm-e2e
+```
+
+`make e2e-test` assumes **KBS / AS / RVPS** images are already loaded into the cluster and deploys with [scenarios/e2e-local-images.yaml](./scenarios/e2e-local-images.yaml). The Makefile injects image repositories, tags, and `pullPolicy: Never` at install time so local runs and CI can use different preloaded image names without changing the scenario file.
+
+- **Local preloaded images**: `trustee-e2e/*:e2e`, `pullPolicy: Never` (not GHCR `:latest`)
+- **CI images**: reuses `docker-e2e-images-linux-amd64` from `workflow-call-build-docker-e2e-materials.yml` as `ghcr.io/confidential-containers/staged-images/*:latest`
+- **Storage**: bundled Postgres KV backend (`storageBackend.type: Postgres`)
+- **KBS sessions**: `sessionStorageType: Memory`
+
+Steps:
+
+1. **`helm-dependency-build`**
+2. **`helm-lint`**
+3. **`deploy`**
+4. **`test-client`** — [e2e/test.sh](./e2e/test.sh)
+5. **`undeploy`** — always attempted, even after failures
+
+Debug individually:
+
+```bash
+make -C deployment/helm-chart load-e2e-images-into-kind
+make -C deployment/helm-chart helm-lint
+make -C deployment/helm-chart deploy
+make -C deployment/helm-chart test-client
+make -C deployment/helm-chart undeploy
+```
+
+Optional variables: `E2E_IMAGE_PREFIX`, `E2E_IMAGE_TAG`, `KBS_CLIENT`.
+
+CI (`test-e2e-kbs.yml`) reuses the Docker Compose e2e image build workflow artifacts: it loads the pre-built Trustee images into kind and uses the pre-built `kbs-client` binary before running Helm e2e.
+
+`make test-client` alone does not build images or undeploy.
+
 ## Development notes
 
 1. Do not change the files under [files](./files/) unless you are updating the corresponding Helm template logic. Service config is rendered from `*.template` files; Postgres KV init SQL lives in `files/postgres-initkv.sql`.
diff --git a/deployment/helm-chart/e2e/fixtures/test-resource.txt b/deployment/helm-chart/e2e/fixtures/test-resource.txt
@@ -0,0 +1 @@
+helm-e2e-trustee-secret-payload
diff --git a/deployment/helm-chart/e2e/test.sh b/deployment/helm-chart/e2e/test.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# kbs-client checks against an already-deployed Trustee Helm release.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+REPO_ROOT="$(cd "${CHART_DIR}/../.." && pwd)"
+
+KBS_CLIENT="${KBS_CLIENT:-${REPO_ROOT}/target/release/kbs-client}"
+KBS_URL="http://127.0.0.1:18080"
+TEST_RESOURCE_FILE="${SCRIPT_DIR}/fixtures/test-resource.txt"
+RESOURCE_PATH="helm-e2e/test-repo/test-secret"
+
+WORK_DIR="$(mktemp -d)"
+ADMIN_TOKEN_FILE="${WORK_DIR}/admin-token"
+ROUNDTRIP_FILE="${WORK_DIR}/roundtrip.txt"
+PORT_FORWARD_PID=""
+
+cleanup() {
+	local code=$?
+	if [[ -n "${PORT_FORWARD_PID}" ]] && kill -0 "${PORT_FORWARD_PID}" 2>/dev/null; then
+		kill "${PORT_FORWARD_PID}" 2>/dev/null || true
+		wait "${PORT_FORWARD_PID}" 2>/dev/null || true
+	fi
+	rm -rf "${WORK_DIR}"
+	exit "${code}"
+}
+trap cleanup EXIT
+
+log() { printf '==> %s\n' "$*"; }
+die() { printf 'error: %s\n' "$*" >&2; exit 1; }
+
+require_cmd() {
+	command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
+}
+
+wait_for_bootstrap_secret() {
+	local secret_name="trustee-e2e-bootstrap-user-keys"
+	local i
+	for i in $(seq 1 120); do
+		if kubectl get secret "${secret_name}" -n coco-trustee-e2e >/dev/null 2>&1; then
+			return 0
+		fi
+		sleep 2
+	done
+	die "timed out waiting for Secret ${secret_name}"
+}
+
+start_port_forward() {
+	log "port-forward KBS -> 127.0.0.1:18080"
+	kubectl port-forward -n coco-trustee-e2e svc/trustee-e2e-kbs 18080:8080 >/dev/null 2>&1 &
+	PORT_FORWARD_PID=$!
+	for _ in $(seq 1 60); do
+		if (echo >/dev/tcp/127.0.0.1/18080) >/dev/null 2>&1; then
+			sleep 2
+			return 0
+		fi
+		sleep 1
+	done
+	die "KBS not reachable on 127.0.0.1:18080 after port-forward"
+}
+
+main() {
+	require_cmd base64
+	require_cmd diff
+
+	[[ -x "${KBS_CLIENT}" ]] || die "kbs-client not found at ${KBS_CLIENT} (set KBS_CLIENT to a pre-built binary)"
+	[[ -f "${TEST_RESOURCE_FILE}" ]] || die "test resource fixture not found: ${TEST_RESOURCE_FILE}"
+
+	wait_for_bootstrap_secret
+	start_port_forward
+
+	kubectl get secret trustee-e2e-bootstrap-user-keys -n coco-trustee-e2e \
+		-o "jsonpath={.data.KBS_ADMIN_TOKEN}" | base64 -d >"${ADMIN_TOKEN_FILE}"
+
+	log "set confidential resource"
+	"${KBS_CLIENT}" --url "${KBS_URL}" config \
+		--admin-token-file "${ADMIN_TOKEN_FILE}" \
+		set-resource \
+		--path "${RESOURCE_PATH}" \
+		--resource-file "${TEST_RESOURCE_FILE}"
+
+	log "set resource policy (allow_all)"
+	"${KBS_CLIENT}" --url "${KBS_URL}" config \
+		--admin-token-file "${ADMIN_TOKEN_FILE}" \
+		set-resource-policy \
+		--allow-all
+
+	log "get resource (expect success with allow_all)"
+	"${KBS_CLIENT}" --url "${KBS_URL}" get-resource \
+		--path "${RESOURCE_PATH}" \
+		| base64 -d >"${ROUNDTRIP_FILE}"
+	diff -u "${TEST_RESOURCE_FILE}" "${ROUNDTRIP_FILE}"
+
+	log "set resource policy (deny_all)"
+	"${KBS_CLIENT}" --url "${KBS_URL}" config \
+		--admin-token-file "${ADMIN_TOKEN_FILE}" \
+		set-resource-policy \
+		--deny-all
+
+	log "get resource (expect failure with deny_all)"
+	if "${KBS_CLIENT}" --url "${KBS_URL}" get-resource \
+		--path "${RESOURCE_PATH}" >/dev/null 2>&1; then
+		die "get-resource succeeded but should have been denied by deny_all resource policy"
+	fi
+
+	log "helm client e2e passed"
+}
+
+main "$@"
diff --git a/deployment/helm-chart/scenarios/e2e-local-images.yaml b/deployment/helm-chart/scenarios/e2e-local-images.yaml
@@ -0,0 +1,20 @@
+# Helm e2e scenario: Postgres KV + Memory KBS sessions.
+# Used by `make e2e-test` (see deployment/helm-chart/Makefile). The Makefile injects
+# image repositories, tags, and pullPolicy so local runs and CI can share this scenario.
+
+sessionStorageType: Memory
+
+storageBackend:
+  type: Postgres
+  postgres:
+    mode: internal
+
+postgresql:
+  enabled: true
+  primary:
+    # The e2e test does not validate persistence; avoid relying on a default
+    # StorageClass in local CI clusters.
+    persistence:
+      enabled: false
+      existingClaim: ""
+      storageClass: ""
