Troubleshooting

# Verify kubectl is installed
which kubectl

# For k3s, ensure KUBECONFIG is set
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml

# Check version (v3+ required)
helm version

# Update Helm if needed
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

# Must be mikefarah/yq v4+
yq --version

# Install correct version
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
chmod +x /usr/local/bin/yq

# Check pod status
kubectl get pods -n control-panel

# Look for pods stuck in Pending or CrashLoopBackOff
kubectl describe pod <pod-name> -n control-panel

kubectl get pvc -n control-panel
kubectl describe pvc <pvc-name> -n control-panel

kubectl describe pod <pod-name> -n control-panel | grep Events

kubectl describe node | grep -A 10 "Allocated resources"

# Check pod logs
kubectl logs <pod-name> -n control-panel

# Check previous container logs if crashing
kubectl logs <pod-name> -n control-panel --previous

# Describe pod for events
kubectl describe pod <pod-name> -n control-panel

kubectl logs cp-pg-postgresql-0 -n control-panel

kubectl get svc cp-cp-ui -n control-panel

kubectl get pods -l app.kubernetes.io/name=cp-ui -n control-panel

kubectl get svc -n control-panel | grep -E "(LoadBalancer|NodePort)"

kubectl port-forward svc/cp-cp-ui 8080:80 -n control-panel --address 0.0.0.0

kubectl get pods -l app.kubernetes.io/name=cp-auth -n control-panel
kubectl logs -l app.kubernetes.io/name=cp-auth -n control-panel

kubectl get pods -l app.kubernetes.io/name=keycloak -n control-panel
kubectl logs cp-keycloak-0 -n control-panel

grep BOOTSTRAP ~/.config/cp-suite/values/cp-*.yaml

kubectl logs -l app.kubernetes.io/name=cp-workflows -n control-panel

kubectl get pods -n control-panel | grep temporal

kubectl describe nodes | grep -A10 "Allocated resources"

kubectl logs -l app.kubernetes.io/name=cp-deployments-api -n control-panel

kubectl get pods -n control-panel-deployments | grep <node-id>

# Execution client (reth)
kubectl logs -f <pod-name> -n control-panel-deployments

# Previous container logs if crashing
kubectl logs <pod-name> -n control-panel-deployments --previous

kubectl get pvc -n control-panel-deployments | grep <node-id>

Warning  PopulatorDataSourceNotFound  ...  Data source control-panel-deployments/cp-<node-id>-reth-snapshot not found

kubectl logs -n control-panel -l app.kubernetes.io/name=cp-workflows --tail=50
kubectl logs -n control-panel -l app.kubernetes.io/name=cp-bolt --tail=50

#!/bin/bash
echo "=== Checking Control Panel Health ==="

echo -e "\n--- Pod Status (Control Panel) ---"
kubectl get pods -n control-panel

echo -e "\n--- Pod Status (Node deployments) ---"
kubectl get pods -n control-panel-deployments

echo -e "\n--- Services ---"
kubectl get svc -n control-panel

echo -e "\n--- PVC Status (Control Panel) ---"
kubectl get pvc -n control-panel

echo -e "\n--- PVC Status (Node deployments) ---"
kubectl get pvc -n control-panel-deployments

echo -e "\n--- Recent Events ---"
kubectl get events -n control-panel --sort-by='.lastTimestamp' | tail -20
kubectl get events -n control-panel-deployments --sort-by='.lastTimestamp' | tail -20

# Node resources
kubectl top nodes

# Pod resources (requires metrics-server)
kubectl top pods -n control-panel

# All pods, last hour
timestamp=$(date +%s)
touch debug-$timestamp.txt
debug=debug-$timestamp.txt
for pod in $(kubectl get pods -n control-panel -o name); do
  echo "=== $pod ===" >> $debug
  kubectl logs $pod -n control-panel --since=1h 2>/dev/null >> $debug
done
for pod in $(kubectl get pods -n control-panel-deployments -o name); do
  echo "=== $pod ===" >> $debug
  kubectl logs $pod -n control-panel-deployments --since=1h 2>/dev/null >> $debug
done

# Restart all deployments
kubectl rollout restart deployment -n control-panel

# Watch pods restart
kubectl get pods -n control-panel -w

# Uninstall completely
cpctl uninstall

# Reinstall
cpctl install

# Check PostgreSQL logs
kubectl logs cp-pg-postgresql-0 -n control-panel

# Check PVC status
kubectl get pvc -n control-panel | grep postgresql

cpctl status > diagnostic.txt
kubectl get events -n control-panel >> diagnostic.txt
kubectl get events -n control-panel-deployments >> diagnostic.txt
timestamp=$(date +%s)
touch debug-$timestamp.txt
debug=debug-$timestamp.txt
for pod in $(kubectl get pods -n control-panel -o name); do
  echo "=== $pod ===" >> $debug
  kubectl logs $pod -n control-panel --since=1h 2>/dev/null >> $debug
done
for pod in $(kubectl get pods -n control-panel-deployments -o name); do
  echo "=== $pod ===" >> $debug
  kubectl logs $pod -n control-panel-deployments --since=1h 2>/dev/null >> $debug
done