Skip to content

Commit b5743ce

Browse files
Xynnn007Xynnn007
committed
helm: add chart e2e test
Add Helm chart e2e targets, a local-image scenario, and kbs-client checks so CI can deploy Trustee on kind with prebuilt materials and validate resource access policy behavior. Assisted-by: Cursor <cursoragent@cursor.com> Signed-off-by: Xynnn007 <xynnn@linux.alibaba.com>
1 parent 109a033 commit b5743ce

7 files changed

Lines changed: 316 additions & 1 deletion

File tree

‎Makefile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
.PHONY: test-all test-unit test-e2e \
22
test-kbs-unit test-as-unit test-trustee-cli-unit \
33
test-kbs-e2e test-as-e2e \
4-
test-kbs-vault-e2e test-kbs-docker-e2e \
4+
test-kbs-vault-e2e test-kbs-docker-e2e test-helm-e2e \
55
kbs-e2e-build kbs-e2e-install-deps kbs-e2e-build-bins \
66
kbs-vault-e2e-install-deps kbs-vault-e2e-run \
77
as-e2e-install-deps as-e2e-run
@@ -95,3 +95,7 @@ as-e2e-run:
9595

9696
test-as-e2e: as-e2e-install-deps as-e2e-run
9797

98+
# Helm chart e2e (deploy Trustee, kbs-client, undeploy; images must already be loaded)
99+
test-helm-e2e:
100+
$(MAKE) -C deployment/helm-chart e2e-test
101+

‎deployment/helm-chart/Makefile‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# Helm chart end-to-end tests (Trustee deploy + kbs-client + undeploy).
2+
#
3+
# Prerequisites: helm, kubectl, base64, diff, and a pre-built kbs-client.
4+
# A Kubernetes cluster must already exist (kind recommended). Cluster creation is out of band.
5+
#
6+
# Images must already be loaded into the target Kubernetes cluster before running e2e-test.
7+
8+
REPO_ROOT := $(abspath ../..)
9+
CHART_DIR := $(abspath .)
10+
KBS_CLIENT ?= $(REPO_ROOT)/target/release/kbs-client
11+
12+
E2E_IMAGE_PREFIX ?= trustee-e2e
13+
E2E_IMAGE_TAG ?= e2e
14+
15+
E2E_KBS_IMAGE_REPOSITORY := $(E2E_IMAGE_PREFIX)/kbs-grpc-as
16+
E2E_AS_IMAGE_REPOSITORY := $(E2E_IMAGE_PREFIX)/coco-as-grpc
17+
E2E_RVPS_IMAGE_REPOSITORY := $(E2E_IMAGE_PREFIX)/rvps
18+
19+
E2E_KBS_IMAGE := $(E2E_KBS_IMAGE_REPOSITORY):$(E2E_IMAGE_TAG)
20+
E2E_AS_IMAGE := $(E2E_AS_IMAGE_REPOSITORY):$(E2E_IMAGE_TAG)
21+
E2E_RVPS_IMAGE := $(E2E_RVPS_IMAGE_REPOSITORY):$(E2E_IMAGE_TAG)
22+
23+
HELM_E2E_IMAGE_ARGS := \
24+
--set-string kbs.image.repository='$(E2E_KBS_IMAGE_REPOSITORY)' \
25+
--set-string kbs.image.tag='$(E2E_IMAGE_TAG)' \
26+
--set-string kbs.image.pullPolicy='Never' \
27+
--set-string as.image.repository='$(E2E_AS_IMAGE_REPOSITORY)' \
28+
--set-string as.image.tag='$(E2E_IMAGE_TAG)' \
29+
--set-string as.image.pullPolicy='Never' \
30+
--set-string rvps.image.repository='$(E2E_RVPS_IMAGE_REPOSITORY)' \
31+
--set-string rvps.image.tag='$(E2E_IMAGE_TAG)' \
32+
--set-string rvps.image.pullPolicy='Never'
33+
34+
export KBS_CLIENT
35+
36+
.PHONY: e2e-test deploy undeploy test-client check-tools wait-ready \
37+
check-docker helm-dependency-build helm-lint load-e2e-images-into-kind
38+
39+
check-tools:
40+
@command -v helm >/dev/null || (echo "missing: helm" >&2; exit 1)
41+
@command -v kubectl >/dev/null || (echo "missing: kubectl" >&2; exit 1)
42+
@command -v base64 >/dev/null || (echo "missing: base64" >&2; exit 1)
43+
@command -v diff >/dev/null || (echo "missing: diff" >&2; exit 1)
44+
45+
check-docker:
46+
@command -v docker >/dev/null || (echo "missing: docker" >&2; exit 1)
47+
48+
helm-dependency-build: check-tools
49+
helm dependency build '$(CHART_DIR)'
50+
51+
helm-lint: helm-dependency-build
52+
helm lint '$(CHART_DIR)' -f '$(CHART_DIR)/scenarios/e2e-local-images.yaml' $(HELM_E2E_IMAGE_ARGS)
53+
54+
load-e2e-images-into-kind: check-docker
55+
@command -v kind >/dev/null || (echo "missing: kind (required to load local images)" >&2; exit 1)
56+
@echo "==> loading images into kind cluster kind"
57+
kind load docker-image --name kind \
58+
'$(E2E_KBS_IMAGE)' '$(E2E_AS_IMAGE)' '$(E2E_RVPS_IMAGE)'
59+
60+
deploy: helm-lint
61+
helm upgrade --install trustee-e2e '$(CHART_DIR)' \
62+
--namespace coco-trustee-e2e \
63+
--create-namespace \
64+
-f '$(CHART_DIR)/scenarios/e2e-local-images.yaml' \
65+
$(HELM_E2E_IMAGE_ARGS) \
66+
--wait \
67+
--timeout 20m
68+
69+
wait-ready: check-tools
70+
kubectl wait --for=condition=available deployment/trustee-e2e-rvps -n coco-trustee-e2e --timeout=600s
71+
kubectl wait --for=condition=available deployment/trustee-e2e-as -n coco-trustee-e2e --timeout=600s
72+
kubectl wait --for=condition=available deployment/trustee-e2e-kbs -n coco-trustee-e2e --timeout=600s
73+
74+
undeploy: check-tools
75+
helm uninstall trustee-e2e -n coco-trustee-e2e 2>/dev/null || true
76+
kubectl delete namespace coco-trustee-e2e --ignore-not-found --timeout=120s 2>/dev/null || true
77+
78+
test-client: check-tools wait-ready
79+
@chmod +x '$(CHART_DIR)/e2e/test.sh'
80+
@'$(CHART_DIR)/e2e/test.sh'
81+
82+
# Deploy, test, then always undeploy. Images must already be present in the cluster.
83+
e2e-test: check-tools
84+
@status=0; \
85+
if [ $$status -eq 0 ]; then \
86+
$(MAKE) deploy || status=$$?; \
87+
fi; \
88+
if [ $$status -eq 0 ]; then \
89+
$(MAKE) test-client || status=$$?; \
90+
fi; \
91+
$(MAKE) undeploy || true; \
92+
exit $$status

‎deployment/helm-chart/README.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,47 @@ Default **`values.yaml`** is intentionally small. Fixed on-disk paths for **Loca
270270
| storageBackend.postgres.mode | string | `"internal"` | Postgres source: `internal` (Bitnami subchart) or `external` (pre-created Secret). |
271271
| storageBackend.type | string | `"LocalFs"` | Backend type: `LocalFs`, `LocalJson`, `Postgres`, or `Memory`. When `Postgres` (or `sessionStorageType` is `Postgres`), the chart injects `POSTGRES_URL`. Only settings for the selected type take effect. |
272272

273+
## End-to-end test
274+
275+
Provision a **kind** cluster out of band, preload the Trustee images into it, point `kubectl` at it, then from the repository root:
276+
277+
```bash
278+
kind create cluster --name kind --wait 5m
279+
make -C deployment/helm-chart load-e2e-images-into-kind
280+
make test-helm-e2e
281+
```
282+
283+
`make e2e-test` assumes **KBS / AS / RVPS** images are already loaded into the cluster and deploys with [scenarios/e2e-local-images.yaml](./scenarios/e2e-local-images.yaml). The Makefile injects image repositories, tags, and `pullPolicy: Never` at install time so local runs and CI can use different preloaded image names without changing the scenario file.
284+
285+
- **Local preloaded images**: `trustee-e2e/*:e2e`, `pullPolicy: Never` (not GHCR `:latest`)
286+
- **CI images**: reuses `docker-e2e-images-linux-amd64` from `workflow-call-build-docker-e2e-materials.yml` as `ghcr.io/confidential-containers/staged-images/*:latest`
287+
- **Storage**: bundled Postgres KV backend (`storageBackend.type: Postgres`)
288+
- **KBS sessions**: `sessionStorageType: Memory`
289+
290+
Steps:
291+
292+
1. **`helm-dependency-build`**
293+
2. **`helm-lint`**
294+
3. **`deploy`**
295+
4. **`test-client`** — [e2e/test.sh](./e2e/test.sh)
296+
5. **`undeploy`** — always attempted, even after failures
297+
298+
Debug individually:
299+
300+
```bash
301+
make -C deployment/helm-chart load-e2e-images-into-kind
302+
make -C deployment/helm-chart helm-lint
303+
make -C deployment/helm-chart deploy
304+
make -C deployment/helm-chart test-client
305+
make -C deployment/helm-chart undeploy
306+
```
307+
308+
Optional variables: `E2E_IMAGE_PREFIX`, `E2E_IMAGE_TAG`, `KBS_CLIENT`.
309+
310+
CI (`test-e2e-kbs.yml`) reuses the Docker Compose e2e image build workflow artifacts: it loads the pre-built Trustee images into kind and uses the pre-built `kbs-client` binary before running Helm e2e.
311+
312+
`make test-client` alone does not build images or undeploy.
313+
273314
## Development notes
274315

275316
1. Do not change the files under [files](./files/) unless you are updating the corresponding Helm template logic. Service config is rendered from `*.template` files; Postgres KV init SQL lives in `files/postgres-initkv.sql`.

‎deployment/helm-chart/README.md.gotmpl‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,47 @@ Default **`values.yaml`** is intentionally small. Fixed on-disk paths for **Loca
159159

160160
{{ template "chart.valuesSection" . }}
161161

162+
## End-to-end test
163+
164+
Provision a **kind** cluster out of band, preload the Trustee images into it, point `kubectl` at it, then from the repository root:
165+
166+
```bash
167+
kind create cluster --name kind --wait 5m
168+
make -C deployment/helm-chart load-e2e-images-into-kind
169+
make test-helm-e2e
170+
```
171+
172+
`make e2e-test` assumes **KBS / AS / RVPS** images are already loaded into the cluster and deploys with [scenarios/e2e-local-images.yaml](./scenarios/e2e-local-images.yaml). The Makefile injects image repositories, tags, and `pullPolicy: Never` at install time so local runs and CI can use different preloaded image names without changing the scenario file.
173+
174+
- **Local preloaded images**: `trustee-e2e/*:e2e`, `pullPolicy: Never` (not GHCR `:latest`)
175+
- **CI images**: reuses `docker-e2e-images-linux-amd64` from `workflow-call-build-docker-e2e-materials.yml` as `ghcr.io/confidential-containers/staged-images/*:latest`
176+
- **Storage**: bundled Postgres KV backend (`storageBackend.type: Postgres`)
177+
- **KBS sessions**: `sessionStorageType: Memory`
178+
179+
Steps:
180+
181+
1. **`helm-dependency-build`**
182+
2. **`helm-lint`**
183+
3. **`deploy`**
184+
4. **`test-client`** — [e2e/test.sh](./e2e/test.sh)
185+
5. **`undeploy`** — always attempted, even after failures
186+
187+
Debug individually:
188+
189+
```bash
190+
make -C deployment/helm-chart load-e2e-images-into-kind
191+
make -C deployment/helm-chart helm-lint
192+
make -C deployment/helm-chart deploy
193+
make -C deployment/helm-chart test-client
194+
make -C deployment/helm-chart undeploy
195+
```
196+
197+
Optional variables: `E2E_IMAGE_PREFIX`, `E2E_IMAGE_TAG`, `KBS_CLIENT`.
198+
199+
CI (`test-e2e-kbs.yml`) reuses the Docker Compose e2e image build workflow artifacts: it loads the pre-built Trustee images into kind and uses the pre-built `kbs-client` binary before running Helm e2e.
200+
201+
`make test-client` alone does not build images or undeploy.
202+
162203
## Development notes
163204

164205
1. Do not change the files under [files](./files/) unless you are updating the corresponding Helm template logic. Service config is rendered from `*.template` files; Postgres KV init SQL lives in `files/postgres-initkv.sql`.

‎deployment/helm-chart/e2e/fixtures/test-resource.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
helm-e2e-trustee-secret-payload

‎deployment/helm-chart/e2e/test.sh‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#!/usr/bin/env bash
2+
# kbs-client checks against an already-deployed Trustee Helm release.
3+
4+
set -euo pipefail
5+
6+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
7+
CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
8+
REPO_ROOT="$(cd "${CHART_DIR}/../.." && pwd)"
9+
10+
KBS_CLIENT="${KBS_CLIENT:-${REPO_ROOT}/target/release/kbs-client}"
11+
KBS_URL="http://127.0.0.1:18080"
12+
TEST_RESOURCE_FILE="${SCRIPT_DIR}/fixtures/test-resource.txt"
13+
RESOURCE_PATH="helm-e2e/test-repo/test-secret"
14+
15+
WORK_DIR="$(mktemp -d)"
16+
ADMIN_TOKEN_FILE="${WORK_DIR}/admin-token"
17+
ROUNDTRIP_FILE="${WORK_DIR}/roundtrip.txt"
18+
PORT_FORWARD_PID=""
19+
20+
cleanup() {
21+
local code=$?
22+
if [[ -n "${PORT_FORWARD_PID}" ]] && kill -0 "${PORT_FORWARD_PID}" 2>/dev/null; then
23+
kill "${PORT_FORWARD_PID}" 2>/dev/null || true
24+
wait "${PORT_FORWARD_PID}" 2>/dev/null || true
25+
fi
26+
rm -rf "${WORK_DIR}"
27+
exit "${code}"
28+
}
29+
trap cleanup EXIT
30+
31+
log() { printf '==> %s\n' "$*"; }
32+
die() { printf 'error: %s\n' "$*" >&2; exit 1; }
33+
34+
require_cmd() {
35+
command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
36+
}
37+
38+
wait_for_bootstrap_secret() {
39+
local secret_name="trustee-e2e-bootstrap-user-keys"
40+
local i
41+
for i in $(seq 1 120); do
42+
if kubectl get secret "${secret_name}" -n coco-trustee-e2e >/dev/null 2>&1; then
43+
return 0
44+
fi
45+
sleep 2
46+
done
47+
die "timed out waiting for Secret ${secret_name}"
48+
}
49+
50+
start_port_forward() {
51+
log "port-forward KBS -> 127.0.0.1:18080"
52+
kubectl port-forward -n coco-trustee-e2e svc/trustee-e2e-kbs 18080:8080 >/dev/null 2>&1 &
53+
PORT_FORWARD_PID=$!
54+
for _ in $(seq 1 60); do
55+
if ! kill -0 "${PORT_FORWARD_PID}" 2>/dev/null; then
56+
die "kubectl port-forward exited unexpectedly"
57+
fi
58+
if (echo >/dev/tcp/127.0.0.1/18080) >/dev/null 2>&1; then
59+
sleep 2
60+
return 0
61+
fi
62+
sleep 1
63+
done
64+
die "KBS not reachable on 127.0.0.1:18080 after port-forward"
65+
}
66+
67+
main() {
68+
require_cmd base64
69+
require_cmd diff
70+
require_cmd kubectl
71+
require_cmd seq
72+
73+
[[ -x "${KBS_CLIENT}" ]] || die "kbs-client not found at ${KBS_CLIENT} (set KBS_CLIENT to a pre-built binary)"
74+
[[ -f "${TEST_RESOURCE_FILE}" ]] || die "test resource fixture not found: ${TEST_RESOURCE_FILE}"
75+
76+
wait_for_bootstrap_secret
77+
start_port_forward
78+
79+
kubectl get secret trustee-e2e-bootstrap-user-keys -n coco-trustee-e2e \
80+
-o "jsonpath={.data.KBS_ADMIN_TOKEN}" | base64 -d >"${ADMIN_TOKEN_FILE}"
81+
82+
log "set confidential resource"
83+
"${KBS_CLIENT}" --url "${KBS_URL}" config \
84+
--admin-token-file "${ADMIN_TOKEN_FILE}" \
85+
set-resource \
86+
--path "${RESOURCE_PATH}" \
87+
--resource-file "${TEST_RESOURCE_FILE}"
88+
89+
log "set resource policy (allow_all)"
90+
"${KBS_CLIENT}" --url "${KBS_URL}" config \
91+
--admin-token-file "${ADMIN_TOKEN_FILE}" \
92+
set-resource-policy \
93+
--allow-all
94+
95+
log "get resource (expect success with allow_all)"
96+
"${KBS_CLIENT}" --url "${KBS_URL}" get-resource \
97+
--path "${RESOURCE_PATH}" \
98+
| base64 -d >"${ROUNDTRIP_FILE}"
99+
diff -u "${TEST_RESOURCE_FILE}" "${ROUNDTRIP_FILE}"
100+
101+
log "set resource policy (deny_all)"
102+
"${KBS_CLIENT}" --url "${KBS_URL}" config \
103+
--admin-token-file "${ADMIN_TOKEN_FILE}" \
104+
set-resource-policy \
105+
--deny-all
106+
107+
log "get resource (expect failure with deny_all)"
108+
if "${KBS_CLIENT}" --url "${KBS_URL}" get-resource \
109+
--path "${RESOURCE_PATH}" >/dev/null 2>&1; then
110+
die "get-resource succeeded but should have been denied by deny_all resource policy"
111+
fi
112+
113+
log "helm client e2e passed"
114+
}
115+
116+
main "$@"

‎deployment/helm-chart/scenarios/e2e-local-images.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Helm e2e scenario: Postgres KV + Memory KBS sessions.
2+
# Used by `make e2e-test` (see deployment/helm-chart/Makefile). The Makefile injects
3+
# image repositories, tags, and pullPolicy so local runs and CI can share this scenario.
4+
5+
sessionStorageType: Memory
6+
7+
storageBackend:
8+
type: Postgres
9+
postgres:
10+
mode: internal
11+
12+
postgresql:
13+
enabled: true
14+
primary:
15+
# The e2e test does not validate persistence; avoid relying on a default
16+
# StorageClass in local CI clusters.
17+
persistence:
18+
enabled: false
19+
existingClaim: ""
20+
storageClass: ""

0 commit comments

Comments
 (0)