Skip to content

Commit 560ae0d

Browse files
pwong2021pwong2021
committed
fix lxd and container pipelines
1 parent 00db38d commit 560ae0d

4 files changed

Lines changed: 75 additions & 162 deletions

File tree

‎.github/scripts/setup-lxd.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env bash
2+
# Install and configure LXD on a GitHub-hosted Ubuntu runner so we can launch
3+
# Ubuntu VMs with IPv4 NAT to the runner's external interface.
4+
#
5+
# Run as root (e.g. `sudo ./setup-lxd.sh`). No env-var inputs.
6+
#
7+
# Not idempotent: `lxd init --auto` errors if LXD is already initialized.
8+
9+
set -euo pipefail
10+
11+
snap install lxd
12+
sleep 15
13+
lxd init --auto
14+
chown ":$SUDO_USER" /var/snap/lxd/common/lxd/unix.socket
15+
16+
# IPv4 NAT bridge with managed DNS
17+
lxc network set lxdbr0 ipv4.address auto
18+
lxc network set lxdbr0 ipv4.nat true
19+
lxc network set lxdbr0 dns.mode managed
20+
lxc network set lxdbr0 dns.domain lxd
21+
22+
# GH runners ship with restrictive FORWARD policy and rp_filter; relax for lxdbr0
23+
sysctl -w net.ipv4.conf.all.forwarding=1 net.ipv4.conf.default.forwarding=1
24+
modprobe br_netfilter
25+
sysctl -w net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1
26+
sysctl -w net.ipv4.conf.all.rp_filter=0 net.ipv4.conf.default.rp_filter=0
27+
iptables -P FORWARD ACCEPT
28+
29+
EXTIF=$(ip -4 route show default | awk '{print $5}' | head -n1)
30+
SUBNET=$(ip -4 addr show dev lxdbr0 | awk '/inet /{print $2}')
31+
iptables -t nat -A POSTROUTING -s "$SUBNET" -o "$EXTIF" -j MASQUERADE
32+
iptables -I FORWARD 1 -i lxdbr0 -o "$EXTIF" -j ACCEPT
33+
iptables -I FORWARD 1 -i "$EXTIF" -o lxdbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT

‎.github/workflows/verify-lxd-vm.yml‎

Lines changed: 36 additions & 162 deletions
Original file line numberDiff line numberDiff line change
@@ -26,59 +26,13 @@ jobs:
2626
- name: Install host packages
2727
run: sudo apt-get update && sudo apt-get -y install jq
2828

29-
- name: Install LXD
30-
run: |
31-
sudo snap install lxd
32-
# Wait for LXD daemon to start
33-
echo "Waiting for LXD daemon to start..."
34-
sleep 15
35-
# Initialize LXD with default settings
36-
sudo lxd init --auto
37-
# Add runner to lxd group
38-
sudo usermod -a -G lxd runner
39-
# Fix socket permissions for CI environment
40-
sudo chmod 666 /var/snap/lxd/common/lxd/unix.socket
41-
42-
echo "Configuring LXD networking (lxdbr0 with IPv4 NAT, IPv6 disabled)"
43-
# Ensure managed bridge exists and is configured for IPv4 NAT and DNS
44-
sudo lxc network show lxdbr0 || true
45-
sudo lxc network set lxdbr0 ipv4.address auto
46-
sudo lxc network set lxdbr0 ipv4.nat true
47-
sudo lxc network set lxdbr0 ipv6.address none || true
48-
sudo lxc network set lxdbr0 ipv6.nat false || true
49-
sudo lxc network set lxdbr0 dns.mode managed || true
50-
sudo lxc network set lxdbr0 dns.domain lxd || true
51-
sudo lxc network set lxdbr0 dns.nameservers 1.1.1.1,8.8.8.8 || true
52-
# Make sure default profile attaches NIC to lxdbr0
53-
sudo lxc profile device add default eth0 nic name=eth0 network=lxdbr0 || true
54-
55-
# Helpful sysctl in some CI hosts (usually already enabled by LXD rules)
56-
sudo sysctl -w net.ipv4.ip_forward=1 || true
57-
sudo sysctl -w net.ipv4.conf.all.forwarding=1 net.ipv4.conf.default.forwarding=1 || true
58-
59-
# Ensure NAT and forwarding rules for lxdbr0 on CI hosts with restrictive FORWARD policies
60-
EXTIF=$(ip -4 route show default | awk '{print $5}' | head -n1)
61-
SUBNET=$(ip -4 addr show dev lxdbr0 | awk '/inet /{print $2}')
62-
echo "Ensuring iptables NAT and FORWARD rules for $SUBNET -> $EXTIF"
63-
sudo iptables -t nat -C POSTROUTING -s "$SUBNET" -o "$EXTIF" -j MASQUERADE 2>/dev/null || sudo iptables -t nat -A POSTROUTING -s "$SUBNET" -o "$EXTIF" -j MASQUERADE
64-
sudo iptables -C FORWARD -i lxdbr0 -o "$EXTIF" -j ACCEPT 2>/dev/null || sudo iptables -I FORWARD 1 -i lxdbr0 -o "$EXTIF" -j ACCEPT
65-
sudo iptables -C FORWARD -i "$EXTIF" -o lxdbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || sudo iptables -I FORWARD 1 -i "$EXTIF" -o lxdbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT
66-
67-
# Some hosts default to DROP on FORWARD; set ACCEPT and enable bridge nf hooks
68-
sudo modprobe br_netfilter || true
69-
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 || true
70-
sudo iptables -P FORWARD ACCEPT || true
71-
72-
# Relax reverse path filtering that can drop forwarded replies in some clouds
73-
sudo sysctl -w net.ipv4.conf.all.rp_filter=0 net.ipv4.conf.default.rp_filter=0 || true
74-
sudo sysctl -w net.ipv4.conf.lxdbr0.rp_filter=0 || true
75-
sudo sysctl -w net.ipv4.conf.$EXTIF.rp_filter=0 || true
76-
77-
# Test basic LXD functionality
78-
sudo lxc list
79-
8029
- name: Check out repository
81-
uses: actions/checkout@v4
30+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
31+
with:
32+
persist-credentials: false
33+
34+
- name: Install and configure LXD
35+
run: sudo ./.github/scripts/setup-lxd.sh
8236

8337
- name: Clone full repository so we can push
8438
run: git fetch --prune --unshallow
@@ -146,136 +100,58 @@ jobs:
146100
ssh_pwauth: false
147101
EOF
148102
149-
sudo lxc launch ${LXD_IMAGE} "$NAME" --vm -c user.user-data="$(cat user-data.yaml)"
103+
sudo lxc launch ${LXD_IMAGE} "$NAME" --vm -c user.user-data="$(cat user-data.yaml)" limits.memory=4GiB
150104
151-
- name: Wait for VM to be ready
105+
- name: Wait for VM
152106
run: |
153-
echo "Waiting for VM to be fully initialized..."
154-
155-
# Debug: Check VM status
156-
echo "=== VM Status ==="
107+
set -euo pipefail
157108
NAME="${{ matrix.suite }}-${{ env.PLATFORM }}"
158-
sudo lxc list
159-
sudo lxc info $NAME
160-
161-
# We check for basic readiness instead of waiting for cloud-init completion
162-
timeout=300
163-
elapsed=0
164-
while [ $elapsed -lt $timeout ]; do
165-
# Debug: Show what we're checking
166-
echo "=== Checking VM readiness (attempt $((elapsed/10 + 1))) ==="
167-
168-
# Check if VM is running and we can execute commands
169-
if sudo lxc list --format=csv | grep -q "$NAME.*RUNNING"; then
170-
echo "VM is in RUNNING state"
171-
172-
# Try to execute a simple command to verify VM is responsive
173-
if sudo lxc exec $NAME -- echo "VM is responsive" 2>/dev/null; then
174-
echo "VM is responsive to commands"
175-
176-
# Check if basic system is ready (systemd, if available)
177-
if sudo lxc exec $NAME -- systemctl is-system-running --wait 2>/dev/null || true; then
178-
echo "VM system is ready!"
179-
break
180-
else
181-
echo "VM system not fully ready yet, but responsive"
182-
# For basic VMs, being responsive might be enough
183-
if [ $elapsed -ge 60 ]; then # After 60 seconds, if responsive, consider ready
184-
echo "VM has been responsive for sufficient time, considering ready"
185-
break
186-
fi
187-
fi
188-
else
189-
echo "VM not responsive to commands yet"
190-
fi
191-
else
192-
echo "VM not in RUNNING state yet"
193-
fi
194-
195-
echo "Waiting for VM to be ready... ($elapsed/$timeout seconds)"
109+
for i in $(seq 1 30); do
110+
lxc list --format=csv | grep -q "$NAME.*RUNNING" && break
196111
sleep 10
197-
elapsed=$((elapsed + 10))
198112
done
113+
lxc list --format=csv | grep -q "$NAME.*RUNNING" || { echo "VM never reached RUNNING"; lxc info "$NAME"; exit 1; }
199114
200-
if [ $elapsed -ge $timeout ]; then
201-
echo "=== TIMEOUT DEBUGGING ==="
202-
echo "Final VM status:"
203-
sudo lxc list
204-
sudo lxc info $NAME
205-
echo "Timeout waiting for VM to be ready"
206-
exit 1
207-
fi
208-
209-
echo "VM is ready for use!"
210-
211-
- name: Export VM IP for Kitchen (KITCHEN_HOST)
115+
- name: Export VM IPv4 for Kitchen
212116
run: |
213117
set -euo pipefail
214118
NAME="${{ matrix.suite }}-${{ env.PLATFORM }}"
215-
216-
# Wait up to 120s for an IP address to appear and scan all interfaces, not just eth0
217-
timeout=120
218-
interval=5
219-
ipv4=""
220-
ipv6=""
221-
elapsed=0
222-
while [ $elapsed -lt $timeout ]; do
223-
ipv4=$(sudo lxc list "$NAME" --format=json | jq -r '.[0].state.network | to_entries[] | .value.addresses[]? | select(.family=="inet" and .scope=="global") | .address' | head -n1 || true)
224-
ipv6=$(sudo lxc list "$NAME" --format=json | jq -r '.[0].state.network | to_entries[] | .value.addresses[]? | select(.family=="inet6" and .scope=="global") | .address' | head -n1 || true)
225-
if [ -n "$ipv4" ] || [ -n "$ipv6" ]; then
226-
break
227-
fi
228-
echo "Waiting for VM IP... ($elapsed/$timeout seconds)"
229-
sleep $interval
230-
elapsed=$((elapsed + interval))
119+
for i in $(seq 1 24); do
120+
ipv4=$(lxc list "$NAME" --format=json | jq -r '.[0].state.network | to_entries[] | .value.addresses[]? | select(.family=="inet" and .scope=="global") | .address' | head -n1)
121+
[ -n "$ipv4" ] && break
122+
sleep 5
231123
done
232-
233-
# Fallback: query IPs from inside the VM if LXD hasn't reported them yet
234-
if [ -z "$ipv4" ] && [ -z "$ipv6" ]; then
235-
echo "Falling back to querying IPs from inside the VM..."
236-
set +e
237-
ipv4=$(sudo lxc exec "$NAME" -- bash -lc "ip -o -4 addr show scope global | awk '{print \\4}' | cut -d/ -f1 | head -n1" 2>/dev/null)
238-
ipv6=$(sudo lxc exec "$NAME" -- bash -lc "ip -o -6 addr show scope global | awk '{print \\4}' | cut -d/ -f1 | head -n1" 2>/dev/null)
239-
set -e
240-
fi
241-
242-
if [ -n "$ipv4" ]; then
243-
echo "Using IPv4 $ipv4 for KITCHEN_HOST"
244-
echo "KITCHEN_HOST=$ipv4" >> "$GITHUB_ENV"
245-
elif [ -n "$ipv6" ]; then
246-
echo "Using IPv6 $ipv6 for KITCHEN_HOST"
247-
echo "KITCHEN_HOST=$ipv6" >> "$GITHUB_ENV"
248-
else
249-
echo "Failed to discover a VM IP address"
250-
sudo lxc list "$NAME"
251-
sudo lxc info "$NAME" || true
252-
exit 1
253-
fi
124+
[ -n "$ipv4" ] || { echo "No VM IPv4 found"; lxc info "$NAME"; exit 1; }
125+
echo "KITCHEN_HOST=$ipv4" >> "$GITHUB_ENV"
254126
255127
- name: Run kitchen test
256128
if: ${{ !contains(steps.commit.outputs.message, 'only-validate-profile') }}
257129
continue-on-error: true
258130
run: bundle exec kitchen test --destroy=always ${{ matrix.suite }}-${{ env.PLATFORM }}
259131

260-
- name: Save our ${{ matrix.suite }} results summary
132+
- name: Save results summary JSON
261133
continue-on-error: true
262-
if: ${{ !contains(steps.commit.outputs.message, 'only-validate-profile') }}
263-
uses: mitre/saf_action@v1.5.2
134+
uses: mitre/saf_action@a19f76ed3721c14aa8a1afb8eded117d8bf16a5d # v1.9.0
264135
with:
265-
command_string: "view summary -j -i spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}.json -o spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}-data.json"
136+
command_string: "view summary -f json -i spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}.json -o spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}-data.json"
266137

267-
- name: Save Test Result JSON
268-
if: ${{ !contains(steps.commit.outputs.message, 'only-validate-profile') }}
269-
uses: actions/upload-artifact@v4
138+
- name: Upload Test Result JSON
139+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
270140
with:
271141
name: ${{ env.PLATFORM }}_${{ matrix.suite }}.json
272142
path: spec/results/
273143

274-
- name: Upload ${{ matrix.suite }} to Heimdall
275-
if: ${{ !contains(steps.commit.outputs.message, 'only-validate-profile') }}
144+
- name: Upload to Heimdall
276145
continue-on-error: true
146+
env:
147+
SAF_HEIMDALL_UPLOAD_KEY: ${{ secrets.SAF_HEIMDALL_UPLOAD_KEY }}
277148
run: |
278-
curl -# -s -F data=@spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}.json -F "filename=${{ env.PLATFORM }}_${{ matrix.suite }}-${{ env.COMMIT_SHORT_SHA }}.json" -F "public=true" -F "evaluationTags=${{ env.COMMIT_SHORT_SHA }},${{ github.repository }},${{ github.workflow }}" -H "Authorization: Api-Key ${{ secrets.SAF_HEIMDALL_UPLOAD_KEY }}" "${{ vars.SAF_HEIMDALL_URL }}/evaluations"
149+
curl -# -s -F data=@spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}.json \
150+
-F "filename=${{ env.PLATFORM }}_${{ matrix.suite }}-${{ env.COMMIT_SHORT_SHA }}.json" \
151+
-F "public=true" \
152+
-F "evaluationTags=${{ env.COMMIT_SHORT_SHA }},${{ github.repository }},${{ github.workflow }}" \
153+
-H "Authorization: Api-Key $SAF_HEIMDALL_UPLOAD_KEY" \
154+
"${{ vars.SAF_HEIMDALL_URL }}/evaluations"
279155
280156
- name: Display our ${{ matrix.suite }} results summary
281157
if: ${{ !contains(steps.commit.outputs.message, 'only-validate-profile') }}
@@ -290,13 +166,11 @@ jobs:
290166
cat spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}-data.json | python markdown-summary.py > spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}-markdown-summary.md
291167
cat spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}-markdown-summary.md >> $GITHUB_STEP_SUMMARY
292168
293-
- name: Ensure the scan meets our ${{ matrix.suite }} results threshold
294-
if: ${{ !contains(steps.commit.outputs.message, 'only-validate-profile') }}
295-
uses: mitre/saf_action@v1.5.2
169+
- name: Validate scan threshold
170+
uses: mitre/saf_action@a19f76ed3721c14aa8a1afb8eded117d8bf16a5d # v1.9.0
296171
with:
297172
command_string: "validate threshold -i spec/results/${{ env.PLATFORM }}_${{ matrix.suite }}.json -F ${{ matrix.suite }}.threshold.yml"
298173

299174
- name: Cleanup ephemeral SSH key
300175
if: always()
301-
run: |
302-
rm -f "$RUNNER_TEMP/kitchen_ed25519" "$RUNNER_TEMP/kitchen_ed25519.pub"
176+
run: rm -f "$RUNNER_TEMP/kitchen_ed25519" "$RUNNER_TEMP/kitchen_ed25519.pub"

‎ubuntu-20_hardened.threshold.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
---
2+
error.total.max: 0
3+
compliance.min: 41

‎ubuntu-20_vanilla.threshold.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
---
2+
error.total.max: 0
3+
compliance.min: 41

0 commit comments

Comments
 (0)