feat(gitops): add safe argocd offline apply workflow

This commit is contained in:
ximenchuifeng committed 2026-07-15 01:15:33 +08:00
1 parent 17e17dc8fa
commit 9d263ebcc0
4 files changed
+196

No files matched your search

+70
View File
@@ -0,0 +1,70 @@
# Argo CD Offline Bootstrap
This directory is the supported installation and recovery path for Argo CD on the demo k3s server `192.168.1.173`.
It allows Argo CD to restart while Harbor `192.168.1.119:8888` is unavailable, provided the exact pinned images are already present in the server's k3s containerd cache. It does not make new or uncached image versions available offline.
## Pinned sources
- Argo CD release: `v3.4.4`
- Upstream manifest: `https://raw.githubusercontent.com/argoproj/argo-cd/v3.4.4/manifests/install.yaml`
- Vendored manifest: `vendor/install-v3.4.4.yaml`
- SHA-256: `b0f9119821f2e19b852c842b9cb235eb9c3ef1549554fbda6aa5904e8d440eae`
The Kustomize overlay rewrites the upstream images to:
```text
192.168.1.119:8888/argoproj/argocd:v3.4.4
192.168.1.119:8888/dexidp/dex:v2.45.0
192.168.1.119:8888/library/redis:8.2.3-alpine
```
All Argo CD Deployment and StatefulSet containers, including init containers, explicitly use `imagePullPolicy: IfNotPresent`.
## Install or recover
Copy this entire directory to the k3s server, then run:
```bash
sudo ./apply.sh
```
The script is idempotent. Before applying anything, it:
1. Renders and verifies the local vendored manifest.
2. Rejects upstream images and any `imagePullPolicy: Always`.
3. Verifies all three exact Harbor image references are in the local k3s image cache.
4. Records the existing Argo CD Applications.
It then applies the render, waits up to 300 seconds per workload, verifies the live pull policies, rejects image-pull failures, and confirms that pre-existing Applications remain present.
The script never pulls, retags, deletes, or garbage-collects images. If an image is missing, restore Harbor or deliberately import that exact image before retrying.
Do not apply the upstream `install.yaml` directly. Its `Always` policies will make Argo CD depend on Harbor again after pods or the server restart.
## Verify
```bash
sudo k3s kubectl -n argocd get pods
sudo k3s kubectl -n argocd get deploy,statefulset \
-o jsonpath='{range .items[*]}{.kind}{"/"}{.metadata.name}{"\n"}{range .spec.template.spec.initContainers[*]} init:{.name}={.imagePullPolicy}{"\n"}{end}{range .spec.template.spec.containers[*]} container:{.name}={.imagePullPolicy}{"\n"}{end}{end}'
sudo k3s kubectl -n argocd get applications.argoproj.io -o name
```
Every listed container and init container must report `IfNotPresent`. No pod may report `ErrImagePull` or `ImagePullBackOff`.
## Upgrade
For each Argo CD upgrade:
1. Download `manifests/install.yaml` from an exact Argo CD release tag, never `stable`.
2. Replace the vendored manifest and record its new SHA-256 here.
3. Update the Argo CD, Dex, and Redis versions in `kustomization.yaml`, `verify-render.sh`, and `apply.sh`.
4. Compare upstream workload and container names with `tests/expected-workloads.txt`; update the inventory and patches if upstream changed them.
5. Run `kubectl kustomize .` and `./verify-render.sh <rendered-file>` locally.
6. Push the new images to Harbor and ensure the exact Harbor references are cached or explicitly imported on `192.168.1.173`.
7. Run only `sudo ./apply.sh`, then verify rollouts, pull policies, pods, and Applications.
Never apply an upgrade until the cache precheck passes.
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
INVENTORY_FILE="$SCRIPT_DIR/tests/expected-workloads.txt"
VERIFY_SCRIPT="$SCRIPT_DIR/verify-render.sh"
ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}"
EXPECTED_IMAGES=(
"192.168.1.119:8888/argoproj/argocd:v3.4.4"
"192.168.1.119:8888/dexidp/dex:v2.45.0"
"192.168.1.119:8888/library/redis:8.2.3-alpine"
)
fail() {
echo "ERROR: $*" >&2
exit 1
}
[[ "${EUID:-$(id -u)}" -eq 0 ]] || fail "this script must run as root (use sudo)"
command -v k3s >/dev/null 2>&1 || fail "k3s is required"
[[ -x "$VERIFY_SCRIPT" ]] || fail "render verifier is missing or not executable: $VERIFY_SCRIPT"
[[ -s "$INVENTORY_FILE" ]] || fail "workload inventory is missing or empty: $INVENTORY_FILE"
temporary_dir="$(mktemp -d)"
trap 'rm -rf "$temporary_dir"' EXIT
rendered_manifest="$temporary_dir/argocd-install.yaml"
cached_images="$temporary_dir/cached-images.txt"
applications_before="$temporary_dir/applications-before.txt"
applications_after="$temporary_dir/applications-after.txt"
echo "Rendering pinned Argo CD offline manifest..."
k3s kubectl kustomize "$SCRIPT_DIR" > "$rendered_manifest" \
|| fail "kustomize render failed"
"$VERIFY_SCRIPT" "$rendered_manifest"
echo "Checking the local k3s containerd image cache..."
k3s ctr images list -q | sort -u > "$cached_images" \
|| fail "could not read the k3s containerd image cache"
missing_images=()
for expected_image in "${EXPECTED_IMAGES[@]}"; do
if ! grep -Fxq "$expected_image" "$cached_images"; then
missing_images+=("$expected_image")
fi
done
if (( ${#missing_images[@]} > 0 )); then
echo "ERROR: required images are missing from the local k3s cache:" >&2
printf ' %s\n' "${missing_images[@]}" >&2
fail "restore Harbor or import the exact images before applying"
fi
if k3s kubectl get crd applications.argoproj.io >/dev/null 2>&1; then
k3s kubectl -n argocd get applications.argoproj.io -o name 2>/dev/null \
| sort > "$applications_before"
else
: > "$applications_before"
fi
echo "Applying the verified manifest..."
k3s kubectl apply -f "$rendered_manifest"
while read -r resource _containers; do
[[ -n "$resource" ]] || continue
resource_kind="${resource%%/*}"
resource_name="${resource#*/}"
resource_kind="$(printf '%s' "$resource_kind" | tr '[:upper:]' '[:lower:]')"
echo "Waiting for $resource_kind/$resource_name..."
k3s kubectl -n argocd rollout status "$resource_kind/$resource_name" \
--timeout="$ROLLOUT_TIMEOUT"
done < "$INVENTORY_FILE"
echo "Verifying live pull policies..."
while read -r resource containers; do
[[ -n "$resource" && -n "$containers" ]] || continue
resource_kind="${resource%%/*}"
resource_name="${resource#*/}"
resource_kind="$(printf '%s' "$resource_kind" | tr '[:upper:]' '[:lower:]')"
live_policies="$temporary_dir/${resource_kind}-${resource_name}-policies.txt"
k3s kubectl -n argocd get "$resource_kind/$resource_name" \
-o jsonpath='{range .spec.template.spec.initContainers[*]}{.name}={.imagePullPolicy}{"\n"}{end}{range .spec.template.spec.containers[*]}{.name}={.imagePullPolicy}{"\n"}{end}' \
| sort > "$live_policies"
expected_count=0
for container in $containers; do
expected_count=$((expected_count + 1))
grep -Fxq "$container=IfNotPresent" "$live_policies" \
|| fail "$resource does not have IfNotPresent for container $container"
done
actual_count="$(wc -l < "$live_policies" | tr -d '[:space:]')"
[[ "$actual_count" -eq "$expected_count" ]] \
|| fail "$resource container inventory changed: expected $expected_count, found $actual_count"
done < "$INVENTORY_FILE"
k3s kubectl -n argocd get applications.argoproj.io -o name 2>/dev/null \
| sort > "$applications_after"
missing_applications="$(comm -23 "$applications_before" "$applications_after")"
if [[ -n "$missing_applications" ]]; then
echo "ERROR: pre-existing Argo CD Applications are missing after apply:" >&2
printf '%s\n' "$missing_applications" >&2
exit 1
fi
pod_waiting_reasons="$(
k3s kubectl -n argocd get pods \
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.initContainerStatuses[*]}{.state.waiting.reason}{" "}{end}{range .status.containerStatuses[*]}{.state.waiting.reason}{" "}{end}{"\n"}{end}'
)"
if printf '%s\n' "$pod_waiting_reasons" | grep -Eq 'ErrImagePull|ImagePullBackOff'; then
printf '%s\n' "$pod_waiting_reasons" >&2
fail "an Argo CD pod is still in an image-pull failure state"
fi
echo "Argo CD offline bootstrap completed successfully."
k3s kubectl -n argocd get pods -o wide
echo "Retained pre-existing Applications: $(wc -l < "$applications_before" | tr -d '[:space:]')"
@@ -4,6 +4,7 @@ kind: Kustomization
namespace: argocd
resources:
- namespace.yaml
- vendor/install-v3.4.4.yaml
images:
+4
View File
@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: argocd