diff --git a/bootstrap/argocd-offline/README.md b/bootstrap/argocd-offline/README.md new file mode 100644 index 0000000..6f46102 --- /dev/null +++ b/bootstrap/argocd-offline/README.md @@ -0,0 +1,70 @@ +# Argo CD Offline Bootstrap + +This directory is the supported installation and recovery path for Argo CD on the demo k3s server `192.168.1.173`. + +It allows Argo CD to restart while Harbor `192.168.1.119:8888` is unavailable, provided the exact pinned images are already present in the server's k3s containerd cache. It does not make new or uncached image versions available offline. + +## Pinned sources + +- Argo CD release: `v3.4.4` +- Upstream manifest: `https://raw.githubusercontent.com/argoproj/argo-cd/v3.4.4/manifests/install.yaml` +- Vendored manifest: `vendor/install-v3.4.4.yaml` +- SHA-256: `b0f9119821f2e19b852c842b9cb235eb9c3ef1549554fbda6aa5904e8d440eae` + +The Kustomize overlay rewrites the upstream images to: + +```text +192.168.1.119:8888/argoproj/argocd:v3.4.4 +192.168.1.119:8888/dexidp/dex:v2.45.0 +192.168.1.119:8888/library/redis:8.2.3-alpine +``` + +All Argo CD Deployment and StatefulSet containers, including init containers, explicitly use `imagePullPolicy: IfNotPresent`. + +## Install or recover + +Copy this entire directory to the k3s server, then run: + +```bash +sudo ./apply.sh +``` + +The script is idempotent. Before applying anything, it: + +1. Renders and verifies the local vendored manifest. +2. Rejects upstream images and any `imagePullPolicy: Always`. +3. Verifies all three exact Harbor image references are in the local k3s image cache. +4. Records the existing Argo CD Applications. + +It then applies the render, waits up to 300 seconds per workload, verifies the live pull policies, rejects image-pull failures, and confirms that pre-existing Applications remain present. + +The script never pulls, retags, deletes, or garbage-collects images. If an image is missing, restore Harbor or deliberately import that exact image before retrying. + +Do not apply the upstream `install.yaml` directly. Its `Always` policies will make Argo CD depend on Harbor again after pods or the server restart. + +## Verify + +```bash +sudo k3s kubectl -n argocd get pods + +sudo k3s kubectl -n argocd get deploy,statefulset \ + -o jsonpath='{range .items[*]}{.kind}{"/"}{.metadata.name}{"\n"}{range .spec.template.spec.initContainers[*]} init:{.name}={.imagePullPolicy}{"\n"}{end}{range .spec.template.spec.containers[*]} container:{.name}={.imagePullPolicy}{"\n"}{end}{end}' + +sudo k3s kubectl -n argocd get applications.argoproj.io -o name +``` + +Every listed container and init container must report `IfNotPresent`. No pod may report `ErrImagePull` or `ImagePullBackOff`. + +## Upgrade + +For each Argo CD upgrade: + +1. Download `manifests/install.yaml` from an exact Argo CD release tag, never `stable`. +2. Replace the vendored manifest and record its new SHA-256 here. +3. Update the Argo CD, Dex, and Redis versions in `kustomization.yaml`, `verify-render.sh`, and `apply.sh`. +4. Compare upstream workload and container names with `tests/expected-workloads.txt`; update the inventory and patches if upstream changed them. +5. Run `kubectl kustomize .` and `./verify-render.sh ` locally. +6. Push the new images to Harbor and ensure the exact Harbor references are cached or explicitly imported on `192.168.1.173`. +7. Run only `sudo ./apply.sh`, then verify rollouts, pull policies, pods, and Applications. + +Never apply an upgrade until the cache precheck passes. diff --git a/bootstrap/argocd-offline/apply.sh b/bootstrap/argocd-offline/apply.sh new file mode 100755 index 0000000..232eff2 --- /dev/null +++ b/bootstrap/argocd-offline/apply.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INVENTORY_FILE="$SCRIPT_DIR/tests/expected-workloads.txt" +VERIFY_SCRIPT="$SCRIPT_DIR/verify-render.sh" +ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}" + +EXPECTED_IMAGES=( + "192.168.1.119:8888/argoproj/argocd:v3.4.4" + "192.168.1.119:8888/dexidp/dex:v2.45.0" + "192.168.1.119:8888/library/redis:8.2.3-alpine" +) + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +[[ "${EUID:-$(id -u)}" -eq 0 ]] || fail "this script must run as root (use sudo)" +command -v k3s >/dev/null 2>&1 || fail "k3s is required" +[[ -x "$VERIFY_SCRIPT" ]] || fail "render verifier is missing or not executable: $VERIFY_SCRIPT" +[[ -s "$INVENTORY_FILE" ]] || fail "workload inventory is missing or empty: $INVENTORY_FILE" + +temporary_dir="$(mktemp -d)" +trap 'rm -rf "$temporary_dir"' EXIT + +rendered_manifest="$temporary_dir/argocd-install.yaml" +cached_images="$temporary_dir/cached-images.txt" +applications_before="$temporary_dir/applications-before.txt" +applications_after="$temporary_dir/applications-after.txt" + +echo "Rendering pinned Argo CD offline manifest..." +k3s kubectl kustomize "$SCRIPT_DIR" > "$rendered_manifest" \ + || fail "kustomize render failed" +"$VERIFY_SCRIPT" "$rendered_manifest" + +echo "Checking the local k3s containerd image cache..." +k3s ctr images list -q | sort -u > "$cached_images" \ + || fail "could not read the k3s containerd image cache" + +missing_images=() +for expected_image in "${EXPECTED_IMAGES[@]}"; do + if ! grep -Fxq "$expected_image" "$cached_images"; then + missing_images+=("$expected_image") + fi +done + +if (( ${#missing_images[@]} > 0 )); then + echo "ERROR: required images are missing from the local k3s cache:" >&2 + printf ' %s\n' "${missing_images[@]}" >&2 + fail "restore Harbor or import the exact images before applying" +fi + +if k3s kubectl get crd applications.argoproj.io >/dev/null 2>&1; then + k3s kubectl -n argocd get applications.argoproj.io -o name 2>/dev/null \ + | sort > "$applications_before" +else + : > "$applications_before" +fi + +echo "Applying the verified manifest..." +k3s kubectl apply -f "$rendered_manifest" + +while read -r resource _containers; do + [[ -n "$resource" ]] || continue + resource_kind="${resource%%/*}" + resource_name="${resource#*/}" + resource_kind="$(printf '%s' "$resource_kind" | tr '[:upper:]' '[:lower:]')" + echo "Waiting for $resource_kind/$resource_name..." + k3s kubectl -n argocd rollout status "$resource_kind/$resource_name" \ + --timeout="$ROLLOUT_TIMEOUT" +done < "$INVENTORY_FILE" + +echo "Verifying live pull policies..." +while read -r resource containers; do + [[ -n "$resource" && -n "$containers" ]] || continue + resource_kind="${resource%%/*}" + resource_name="${resource#*/}" + resource_kind="$(printf '%s' "$resource_kind" | tr '[:upper:]' '[:lower:]')" + live_policies="$temporary_dir/${resource_kind}-${resource_name}-policies.txt" + + k3s kubectl -n argocd get "$resource_kind/$resource_name" \ + -o jsonpath='{range .spec.template.spec.initContainers[*]}{.name}={.imagePullPolicy}{"\n"}{end}{range .spec.template.spec.containers[*]}{.name}={.imagePullPolicy}{"\n"}{end}' \ + | sort > "$live_policies" + + expected_count=0 + for container in $containers; do + expected_count=$((expected_count + 1)) + grep -Fxq "$container=IfNotPresent" "$live_policies" \ + || fail "$resource does not have IfNotPresent for container $container" + done + + actual_count="$(wc -l < "$live_policies" | tr -d '[:space:]')" + [[ "$actual_count" -eq "$expected_count" ]] \ + || fail "$resource container inventory changed: expected $expected_count, found $actual_count" +done < "$INVENTORY_FILE" + +k3s kubectl -n argocd get applications.argoproj.io -o name 2>/dev/null \ + | sort > "$applications_after" + +missing_applications="$(comm -23 "$applications_before" "$applications_after")" +if [[ -n "$missing_applications" ]]; then + echo "ERROR: pre-existing Argo CD Applications are missing after apply:" >&2 + printf '%s\n' "$missing_applications" >&2 + exit 1 +fi + +pod_waiting_reasons="$( + k3s kubectl -n argocd get pods \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.initContainerStatuses[*]}{.state.waiting.reason}{" "}{end}{range .status.containerStatuses[*]}{.state.waiting.reason}{" "}{end}{"\n"}{end}' +)" +if printf '%s\n' "$pod_waiting_reasons" | grep -Eq 'ErrImagePull|ImagePullBackOff'; then + printf '%s\n' "$pod_waiting_reasons" >&2 + fail "an Argo CD pod is still in an image-pull failure state" +fi + +echo "Argo CD offline bootstrap completed successfully." +k3s kubectl -n argocd get pods -o wide +echo "Retained pre-existing Applications: $(wc -l < "$applications_before" | tr -d '[:space:]')" diff --git a/bootstrap/argocd-offline/kustomization.yaml b/bootstrap/argocd-offline/kustomization.yaml index fde7ef0..ea69a2b 100644 --- a/bootstrap/argocd-offline/kustomization.yaml +++ b/bootstrap/argocd-offline/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization namespace: argocd resources: + - namespace.yaml - vendor/install-v3.4.4.yaml images: diff --git a/bootstrap/argocd-offline/namespace.yaml b/bootstrap/argocd-offline/namespace.yaml new file mode 100644 index 0000000..a040f2b --- /dev/null +++ b/bootstrap/argocd-offline/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: argocd