feat(gitops): add safe argocd offline apply workflow
This commit is contained in:
1 parent
17e17dc8fa
commit
9d263ebcc0
4 files changed
+196
No files matched your search
@@ -0,0 +1,70 @@
|
|||||||
|
# Argo CD Offline Bootstrap
|
||||||
|
|
||||||
|
This directory is the supported installation and recovery path for Argo CD on the demo k3s server `192.168.1.173`.
|
||||||
|
|
||||||
|
It allows Argo CD to restart while Harbor `192.168.1.119:8888` is unavailable, provided the exact pinned images are already present in the server's k3s containerd cache. It does not make new or uncached image versions available offline.
|
||||||
|
|
||||||
|
## Pinned sources
|
||||||
|
|
||||||
|
- Argo CD release: `v3.4.4`
|
||||||
|
- Upstream manifest: `https://raw.githubusercontent.com/argoproj/argo-cd/v3.4.4/manifests/install.yaml`
|
||||||
|
- Vendored manifest: `vendor/install-v3.4.4.yaml`
|
||||||
|
- SHA-256: `b0f9119821f2e19b852c842b9cb235eb9c3ef1549554fbda6aa5904e8d440eae`
|
||||||
|
|
||||||
|
The Kustomize overlay rewrites the upstream images to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
192.168.1.119:8888/argoproj/argocd:v3.4.4
|
||||||
|
192.168.1.119:8888/dexidp/dex:v2.45.0
|
||||||
|
192.168.1.119:8888/library/redis:8.2.3-alpine
|
||||||
|
```
|
||||||
|
|
||||||
|
All Argo CD Deployment and StatefulSet containers, including init containers, explicitly use `imagePullPolicy: IfNotPresent`.
|
||||||
|
|
||||||
|
## Install or recover
|
||||||
|
|
||||||
|
Copy this entire directory to the k3s server, then run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./apply.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The script is idempotent. Before applying anything, it:
|
||||||
|
|
||||||
|
1. Renders and verifies the local vendored manifest.
|
||||||
|
2. Rejects upstream images and any `imagePullPolicy: Always`.
|
||||||
|
3. Verifies all three exact Harbor image references are in the local k3s image cache.
|
||||||
|
4. Records the existing Argo CD Applications.
|
||||||
|
|
||||||
|
It then applies the render, waits up to 300 seconds per workload, verifies the live pull policies, rejects image-pull failures, and confirms that pre-existing Applications remain present.
|
||||||
|
|
||||||
|
The script never pulls, retags, deletes, or garbage-collects images. If an image is missing, restore Harbor or deliberately import that exact image before retrying.
|
||||||
|
|
||||||
|
Do not apply the upstream `install.yaml` directly. Its `Always` policies will make Argo CD depend on Harbor again after pods or the server restart.
|
||||||
|
|
||||||
|
## Verify
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo k3s kubectl -n argocd get pods
|
||||||
|
|
||||||
|
sudo k3s kubectl -n argocd get deploy,statefulset \
|
||||||
|
-o jsonpath='{range .items[*]}{.kind}{"/"}{.metadata.name}{"\n"}{range .spec.template.spec.initContainers[*]} init:{.name}={.imagePullPolicy}{"\n"}{end}{range .spec.template.spec.containers[*]} container:{.name}={.imagePullPolicy}{"\n"}{end}{end}'
|
||||||
|
|
||||||
|
sudo k3s kubectl -n argocd get applications.argoproj.io -o name
|
||||||
|
```
|
||||||
|
|
||||||
|
Every listed container and init container must report `IfNotPresent`. No pod may report `ErrImagePull` or `ImagePullBackOff`.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
For each Argo CD upgrade:
|
||||||
|
|
||||||
|
1. Download `manifests/install.yaml` from an exact Argo CD release tag, never `stable`.
|
||||||
|
2. Replace the vendored manifest and record its new SHA-256 here.
|
||||||
|
3. Update the Argo CD, Dex, and Redis versions in `kustomization.yaml`, `verify-render.sh`, and `apply.sh`.
|
||||||
|
4. Compare upstream workload and container names with `tests/expected-workloads.txt`; update the inventory and patches if upstream changed them.
|
||||||
|
5. Run `kubectl kustomize .` and `./verify-render.sh <rendered-file>` locally.
|
||||||
|
6. Push the new images to Harbor and ensure the exact Harbor references are cached or explicitly imported on `192.168.1.173`.
|
||||||
|
7. Run only `sudo ./apply.sh`, then verify rollouts, pull policies, pods, and Applications.
|
||||||
|
|
||||||
|
Never apply an upgrade until the cache precheck passes.
|
||||||
Executable
+121
@@ -0,0 +1,121 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
INVENTORY_FILE="$SCRIPT_DIR/tests/expected-workloads.txt"
|
||||||
|
VERIFY_SCRIPT="$SCRIPT_DIR/verify-render.sh"
|
||||||
|
ROLLOUT_TIMEOUT="${ROLLOUT_TIMEOUT:-300s}"
|
||||||
|
|
||||||
|
EXPECTED_IMAGES=(
|
||||||
|
"192.168.1.119:8888/argoproj/argocd:v3.4.4"
|
||||||
|
"192.168.1.119:8888/dexidp/dex:v2.45.0"
|
||||||
|
"192.168.1.119:8888/library/redis:8.2.3-alpine"
|
||||||
|
)
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
echo "ERROR: $*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
[[ "${EUID:-$(id -u)}" -eq 0 ]] || fail "this script must run as root (use sudo)"
|
||||||
|
command -v k3s >/dev/null 2>&1 || fail "k3s is required"
|
||||||
|
[[ -x "$VERIFY_SCRIPT" ]] || fail "render verifier is missing or not executable: $VERIFY_SCRIPT"
|
||||||
|
[[ -s "$INVENTORY_FILE" ]] || fail "workload inventory is missing or empty: $INVENTORY_FILE"
|
||||||
|
|
||||||
|
temporary_dir="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$temporary_dir"' EXIT
|
||||||
|
|
||||||
|
rendered_manifest="$temporary_dir/argocd-install.yaml"
|
||||||
|
cached_images="$temporary_dir/cached-images.txt"
|
||||||
|
applications_before="$temporary_dir/applications-before.txt"
|
||||||
|
applications_after="$temporary_dir/applications-after.txt"
|
||||||
|
|
||||||
|
echo "Rendering pinned Argo CD offline manifest..."
|
||||||
|
k3s kubectl kustomize "$SCRIPT_DIR" > "$rendered_manifest" \
|
||||||
|
|| fail "kustomize render failed"
|
||||||
|
"$VERIFY_SCRIPT" "$rendered_manifest"
|
||||||
|
|
||||||
|
echo "Checking the local k3s containerd image cache..."
|
||||||
|
k3s ctr images list -q | sort -u > "$cached_images" \
|
||||||
|
|| fail "could not read the k3s containerd image cache"
|
||||||
|
|
||||||
|
missing_images=()
|
||||||
|
for expected_image in "${EXPECTED_IMAGES[@]}"; do
|
||||||
|
if ! grep -Fxq "$expected_image" "$cached_images"; then
|
||||||
|
missing_images+=("$expected_image")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if (( ${#missing_images[@]} > 0 )); then
|
||||||
|
echo "ERROR: required images are missing from the local k3s cache:" >&2
|
||||||
|
printf ' %s\n' "${missing_images[@]}" >&2
|
||||||
|
fail "restore Harbor or import the exact images before applying"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if k3s kubectl get crd applications.argoproj.io >/dev/null 2>&1; then
|
||||||
|
k3s kubectl -n argocd get applications.argoproj.io -o name 2>/dev/null \
|
||||||
|
| sort > "$applications_before"
|
||||||
|
else
|
||||||
|
: > "$applications_before"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Applying the verified manifest..."
|
||||||
|
k3s kubectl apply -f "$rendered_manifest"
|
||||||
|
|
||||||
|
while read -r resource _containers; do
|
||||||
|
[[ -n "$resource" ]] || continue
|
||||||
|
resource_kind="${resource%%/*}"
|
||||||
|
resource_name="${resource#*/}"
|
||||||
|
resource_kind="$(printf '%s' "$resource_kind" | tr '[:upper:]' '[:lower:]')"
|
||||||
|
echo "Waiting for $resource_kind/$resource_name..."
|
||||||
|
k3s kubectl -n argocd rollout status "$resource_kind/$resource_name" \
|
||||||
|
--timeout="$ROLLOUT_TIMEOUT"
|
||||||
|
done < "$INVENTORY_FILE"
|
||||||
|
|
||||||
|
echo "Verifying live pull policies..."
|
||||||
|
while read -r resource containers; do
|
||||||
|
[[ -n "$resource" && -n "$containers" ]] || continue
|
||||||
|
resource_kind="${resource%%/*}"
|
||||||
|
resource_name="${resource#*/}"
|
||||||
|
resource_kind="$(printf '%s' "$resource_kind" | tr '[:upper:]' '[:lower:]')"
|
||||||
|
live_policies="$temporary_dir/${resource_kind}-${resource_name}-policies.txt"
|
||||||
|
|
||||||
|
k3s kubectl -n argocd get "$resource_kind/$resource_name" \
|
||||||
|
-o jsonpath='{range .spec.template.spec.initContainers[*]}{.name}={.imagePullPolicy}{"\n"}{end}{range .spec.template.spec.containers[*]}{.name}={.imagePullPolicy}{"\n"}{end}' \
|
||||||
|
| sort > "$live_policies"
|
||||||
|
|
||||||
|
expected_count=0
|
||||||
|
for container in $containers; do
|
||||||
|
expected_count=$((expected_count + 1))
|
||||||
|
grep -Fxq "$container=IfNotPresent" "$live_policies" \
|
||||||
|
|| fail "$resource does not have IfNotPresent for container $container"
|
||||||
|
done
|
||||||
|
|
||||||
|
actual_count="$(wc -l < "$live_policies" | tr -d '[:space:]')"
|
||||||
|
[[ "$actual_count" -eq "$expected_count" ]] \
|
||||||
|
|| fail "$resource container inventory changed: expected $expected_count, found $actual_count"
|
||||||
|
done < "$INVENTORY_FILE"
|
||||||
|
|
||||||
|
k3s kubectl -n argocd get applications.argoproj.io -o name 2>/dev/null \
|
||||||
|
| sort > "$applications_after"
|
||||||
|
|
||||||
|
missing_applications="$(comm -23 "$applications_before" "$applications_after")"
|
||||||
|
if [[ -n "$missing_applications" ]]; then
|
||||||
|
echo "ERROR: pre-existing Argo CD Applications are missing after apply:" >&2
|
||||||
|
printf '%s\n' "$missing_applications" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
pod_waiting_reasons="$(
|
||||||
|
k3s kubectl -n argocd get pods \
|
||||||
|
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.initContainerStatuses[*]}{.state.waiting.reason}{" "}{end}{range .status.containerStatuses[*]}{.state.waiting.reason}{" "}{end}{"\n"}{end}'
|
||||||
|
)"
|
||||||
|
if printf '%s\n' "$pod_waiting_reasons" | grep -Eq 'ErrImagePull|ImagePullBackOff'; then
|
||||||
|
printf '%s\n' "$pod_waiting_reasons" >&2
|
||||||
|
fail "an Argo CD pod is still in an image-pull failure state"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Argo CD offline bootstrap completed successfully."
|
||||||
|
k3s kubectl -n argocd get pods -o wide
|
||||||
|
echo "Retained pre-existing Applications: $(wc -l < "$applications_before" | tr -d '[:space:]')"
|
||||||
@@ -4,6 +4,7 @@ kind: Kustomization
|
|||||||
namespace: argocd
|
namespace: argocd
|
||||||
|
|
||||||
resources:
|
resources:
|
||||||
|
- namespace.yaml
|
||||||
- vendor/install-v3.4.4.yaml
|
- vendor/install-v3.4.4.yaml
|
||||||
|
|
||||||
images:
|
images:
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: argocd
|
||||||
Reference in new issue
Block a user