# Re-run this script when creating a new worker!

# Run on the worker
# We need to set nvidia runtime as the default runtime for containerd.
sudo tee /var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.tmpl >/dev/null <<'EOF'
{{ template "base" . }}

[plugins."io.containerd.cri.v1.runtime".containerd]
  default_runtime_name = "nvidia"
EOF

sudo systemctl restart k3s-agent

# Ensure the main autogenerated config (/var/lib/rancher/k3s/agent/etc/containerd/config.toml)
#  has  values like
#
# [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'nvidia']
#   runtime_type = "io.containerd.runc.v2"

# [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'nvidia'.options]
#   BinaryName = "/usr/bin/nvidia-container-runtime"
#   SystemdCgroup = true

# ...
# [plugins."io.containerd.cri.v1.runtime".containerd]
#   default_runtime_name = "nvidia"

# We configured only the default_runtime_name, everything else is expected to be autogenerated by k3s


# Run on the k8s-control-plane node

# Apply the NVIDIA device plugin manifest
# Check for the latest stable version from: https://github.com/NVIDIA/k8s-device-plugin
kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.17.1/deployments/static/nvidia-device-plugin.yml

# Run a gpu pod to test:
# https://github.com/NVIDIA/k8s-device-plugin?tab=readme-ov-file#running-gpu-jobs

# TROUBLESHOOTING:

# 1. Configs should be the same on all worker nodes:
#   sudo cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml
# 2. Strangely even after providing the right config and restarting the agent the nvidia plugin still couldn't see the device
#   What helped:
#   a. On control plane:
#      kubectl delete -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.17.1/deployments/static/nvidia-device-plugin.yml
#   b. On every worker:
#      sudo systemctl restart k3s-agent
#   c. On control plane:
#      kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.17.1/deployments/static/nvidia-device-plugin.yml
#
#   Maybe next time if there's an issue like that we should just delete individual plugin pods instead of deleting the whole plugin
