Refactor AKS GPU consumption to use separate images per GPU driver ty…

…pe (#5133)

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -148,8 +148,8 @@ export GPU_DEST=/usr/local/nvidia
 DOCKER_VERSION=1.13.1-1
 export NVIDIA_DRIVER_IMAGE_SHA="${GPU_IMAGE_SHA:=}"
 export NVIDIA_DRIVER_IMAGE_TAG="${GPU_DV}-${NVIDIA_DRIVER_IMAGE_SHA}"
-export NVIDIA_DRIVER_IMAGE="mcr.microsoft.com/aks/aks-gpu"
 export NVIDIA_GPU_DRIVER_TYPE="${GPU_DRIVER_TYPE:=}"
+export NVIDIA_DRIVER_IMAGE="mcr.microsoft.com/aks/aks-gpu-${NVIDIA_GPU_DRIVER_TYPE}"
 export CTR_GPU_INSTALL_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind"
 export DOCKER_GPU_INSTALL_CMD="docker run --privileged --net=host --pid=host -v /opt/gpu:/mnt/gpu -v /opt/actions:/mnt/actions --rm"
 APT_CACHE_DIR=/var/cache/apt/archives/

pkg/agent/baker.go

@@ -1058,9 +1058,9 @@ func useGridDrivers(size string) bool {
 
 func getAKSGPUImageSHA(size string) string {
 	if useGridDrivers(size) {
-		return datamodel.AKSGPUGridSHA
+		return datamodel.AKSGPUGridVersionSuffix
 	}
-	return datamodel.AKSGPUCudaSHA
+	return datamodel.AKSGPUCudaVersionSuffix
 }
 
 func getGPUDriverType(size string) string {

pkg/agent/baker_test.go

@@ -2169,10 +2169,10 @@ var _ = Describe("getGPUDriverType", func() {
 })
 
 var _ = Describe("getAKSGPUImageSHA", func() {
-	It("should use newest AKSGPUGridSHA with nv v5", func() {
-		Expect(getAKSGPUImageSHA("standard_nv6ads_a10_v5")).To(Equal(datamodel.AKSGPUGridSHA))
+	It("should use newest AKSGPUGridVersionSuffix with nv v5", func() {
+		Expect(getAKSGPUImageSHA("standard_nv6ads_a10_v5")).To(Equal(datamodel.AKSGPUGridVersionSuffix))
 	})
-	It("should use newest AKSGPUCudaSHA with non grid SKU", func() {
-		Expect(getAKSGPUImageSHA("standard_nc6_v3")).To(Equal(datamodel.AKSGPUCudaSHA))
+	It("should use newest AKSGPUCudaVersionSuffix with non grid SKU", func() {
+		Expect(getAKSGPUImageSHA("standard_nc6_v3")).To(Equal(datamodel.AKSGPUCudaVersionSuffix))
 	})
 })

pkg/agent/datamodel/const.go

@@ -134,15 +134,15 @@ const (
 
 const (
 	Nvidia470CudaDriverVersion = "cuda-470.82.01"
-	Nvidia550CudaDriverVersion = "cuda-550.90.12"
-	Nvidia535GridDriverVersion = "grid-535.161.08"
+	Nvidia550CudaDriverVersion = "550.90.12"
+	Nvidia535GridDriverVersion = "535.161.08"
 )
 
 // These SHAs will change once we update aks-gpu images in aks-gpu repository. We do that fairly rarely at this time.
 // So for now these will be kept here like this.
 const (
-	AKSGPUCudaSHA = "sha-c66998"
-	AKSGPUGridSHA = "sha-c66998"
+	AKSGPUCudaVersionSuffix = "20241021235610"
+	AKSGPUGridVersionSuffix = "20241021235607"
 )
 
 /* convergedGPUDriverSizes : these sizes use a "converged" driver to support both cuda/grid workloads.

pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData

@@ -6,7 +6,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAC/9xc/1ejSLb/PX/FHYYZdUaSEG3bsZfewQRtnvn2gNjTz3Y5GCqRkUAaCrtdzfvb36migIIQW9vZebPrnj6L1K17b33u16rC+f671pUXtK6c+LqhGYZtfjAtbdC1+rZpqYZln6h6X9kHOtbtjyY9Wx/qlm3pA200sZRX6ciJ3tfs96rVfZePHKQj70b9nv1e7evDyW/qqTa0lNfpgKH1NdXUaggOUwJ1bNn60LTUfj9n+ks61Bt1zzRjbbTTLg33Ru+H/ZHaK8bl0viZ9qGGplOiIUoQunx4rzTMYdRhIA1Gxx/otL5uFjh1GFAD0z4dn24QfcBxWFsbg607GlqqPtSM3joJA86YDLvrgwy6s0OTEAz14Wk+ttcuxta02mOgnU2ONeIWw5Fln4wmw56yx6DSB6c10/aKaX3N4pHa2y9bcXBqjyecqnuv1pbateroDtbpDJ3ouE66GT3qutyiDtcozzXD1EdDWx+eq329p+wxKLtDfX3d+3Ju6LEx6tk97biGqLNOND47tdVejwUcQ29kqJxJNMMYGco+gyeNVG4hdOJh5kO9sTE61thbpvFoYh2TVZLVDdOhV5zt1bFum5pxrhk8gVxL0Buadn80OpuMGVmnlkz9n4mhrRNnzqFaan0sHLQ5imoQHsjcYNXRD5gi5+96VdserDvWmuDXbS6I1keZ5O7EtEYD29RUo/vO7o0Gqj4007UdMgan48n6/MM9btDQiWfxkXG4vz5cXd/hK45GO9e7mj3uT071YYnRQUGkD09GtjEa2N2RYUzGltZTDlk4mKe/bZTzS3udhpPwi1xk6Z6q9wvr/MLl78m4p1paMZRFjqkRxz/XaVyZXUMfW9ROhqb2Ck5yu80JIel0Mj411J7GUXBqjCfGKXN4ud3Jg4QkBEMjZlDkNsO/q5u2apr66dA2RiPLHr9XZFleG6P+M9aMgW4STRVZZmzHKk1d3dH4AyVSZJnnPB73P9hj1TTfjwwabCf6qSLLlcDNa8iIhYUsZxktTWNr7iPLr0sEo7FmqBbBME0NspxlL8uoGfwFGilYNChNS+2e2aeaZavGwLZGZ9pQkbMSWiUZatb7kXHG1jJJGStyVlGr5ObkeKhZ9tjQTvTfFJkU1Twojyd6P8tlMqmLKSjv1bHdNTTiLikYWVXiR/ShOTk50bu6NqQecWabY7WrKTKpUpTa0vraeGRYvVLaVOQszxXjmcOnw9lCDEs/UbukXhmaOiBVMuOjyFmOqyFizBSZpLa08bGsMXHy3z7YXZXgdq4ZliJniY1FBhnSDMtU5IO8OWFqm1qXgGr1Tft4NLJMy1DHeTnVftO6Wdzn3nGQGbinm8dqX7P1saUe9zVTkUlaS3OmMTEtwmHdubLk1tNNMss2zXeK/DrzeEMfqMYHe6h3bX3MJVX5NfN8fWhqhmXrg55pG5ppERcl3mdMiCZDa2QP1OFpX7OpTor8ev+p8070vqUZ+TwWRj0ChLZh3glJeBV5B0+dV5H3mnNeQyO2sA3tvye6ofWUTpaihiOaFNRTzWTAdLLcZA7VsflulGdDvmlsZ5ExNvRzMkjqJ2kFiFd2snRFXvLu2mnvZ2GTt+sD1TxjTNlY19B62tDS1X6abHu1PXE7i0DSzlRbnU77ABqNohWhTRXRhmn/Gipj5D1BlVR/W1Y67cMqBVd836vmQOm0f3mUyZ7SkduPUuwrnSx3b6B4pXTkDBXWh2ROnvenmnFOAplEo36id4ktjJGVJlBLPVU68l6j4c1g5gUutBCegoTvlghmuz5IgbNAIPwkRchHTowEkJaRF2CQPiUehs7blotuW0Hi+/AA8wgtQfoEW82tN4CvUdAAABiZirgdhxEGKaLsWzk3Msf5fANbCwdPr7fF9i60/rGt9+y+fqYp29MwQmG886D3lO3mTzs7YmsXnB24h1QFHCbLJYq2nYvOJTgXe5c7bwB98TCstnaY5Mzsz9Qgd5ZnC4YHwBFILmwJWzsN5MeIaoKm1yEIFdFBiGEWJoErNGZeozE5ngytiT0y7aE60BQh/V1oDFSDOFUxwF4UI7RbrA6Tl0KDli+6Ay0IindCg+19lFYSRy0/nDo+3TTfJFdoiv1GWsjTUfLeDac3KGqgL0uCJm3UzhVBvC/augzxI2UllOg00+KlBLee6zlMQG4muSnvNWVJzmYOz/Wermas9YF6qtnmOzUTmb/gpdXNIW6eqXm+ksT7DYwf5aIIi2nUXHjTKIzDGW5Ow0XLuYnJP2m+TKpTOUysD2OtAhN5xWtN+pq0p02TYXfQU4Qp8aUAbg7jphdClAQgkfC79Xw0Ry5IUrQASQoQlq7DGIMkffbwtRTEsPTco9YyCqctuRXEraVHiBdhQpz3bomUKy9wd+NoqrTCJW7Nl8muG2OltQjSX8Il9sIgViJC99hMZ0oJi9nZixKHfJHM2mvrTN2qboEBwgpb29Jz2eMtZGofZSrnL5kCR7w2FCihQTrprtp9p9k93VBat07UmjrTa9RyyLxoeu3dorjVIF2xOiRtGEcchSEmdOmEhnauDS2y8zs9pY1Uxs8P5y3nn0mEWoPcTVTye1P7glEQE22a3STG4cKcRt4St9AtCnDcanQnRp/sYscTS2nhxbI1TSLfvkXRVRijZpjgBq0APEUYOfE6haGd6qZlfGANbdrE08xD6FvTMJh58+ads/Ch0YgQju6mC9f2ZvbM8fwkQts7cE9zFhnzUKyI8hv47HjYjn2ElorYeQPYW6AwwYq49wbia2+G4ccfKw+UxSyMwAMvAHE7Rp9ABpEx3XkDbkhJyA9jB2L2IIj3v64EwusqQs4NPDzAx5zam8EFiB5I6FPODy65gpP90KSrfUHTBCMXPgrirx8FMpFIid+USCOEkygAOX+ZZ+7shy4exAKIfHTm0Uc3DLhcv1nsqgA9iP0wvEmWOeQ8zDIHM4E8xI5v88BHaBpGriLu06kxdiJMxxVx23Uwgp9/iNMy+Pna8xHgKEEl1DPpIOVcM/wlqqLSJvgSIRssMU2iCAUbpBam2t7mCUHidN3ZAWmOQCytbrMxBYvQcf5SmhYhEpzu7rNWRrweucLj3sBMvNkPcuM/Ckm6htw1nqVmGH0NyBhNw8CNhZKLFXFtB6EdYwfHf4UAJ/H90vD+o2KWQ2uOsI2d6Mrx/Rwl7ET2Y0il5BSpJPKzcExNLd5zs1cZ4MIG6DjaCnxOBBL+54ySEGlPgY7j9o3wZcY6aBO39kGaxWb/FkT6HBbKvAWRq17QefujXOJD1LoA8e/wnQJtuKxThoaOg0uMSgRcBL7UsvYsChd2hOZejKM7m7RMNimNL7Q4mTULfRdFJO69KN04ZSAJO/+JbkFggyXZ/PFOwWAgXRcDWUq7DhDvN7YoK+JGXIvzMjfiGP3hbnTlBU5091UvYmSPOBKjWDr4uupMmYx1f+ImVXyqLLDiVht8qzyn7F7UbaRZRWat01Cn2+wxJQcsi4RNZlzzw1rOX/fHEpJ/tEs+wy2/6po17vm4i1YmbPBYuo2YeX7R19M3T6z9ZGLqn/vMP1+VfI7n9dVExhOXXe2COpqYSYPLyzyV1WcxntUL01jek9TWuFwlYnt4AIxQudoVB2B/lZpHCWZhVLb6IwYvjLxHSZfOLXLZ/p7uZad+mLiSF3hYIrRxk1JQ2vTI7+QTcYaMz0oAkVIQIzJDtJ/TLuZMt77XRidbnBUe94o/o1uklMgFyQOhRdRruUKhYBEchc5v3zI4GqtGfBdjtJhi4ry0eX9eMw7x7ZTUgaxMfGv7nasBroMWYSBFyA8d9ykTmN4gMlX+hC4+l022L0lciJakIJSWzpwk9nL4/R4mUeD4ZJKU5BO+KZwKk8U4XP7b2Yso/Wca69moul7sXPno3w5YpvdfE9s0w5ClPA/Xl4JJA05KQfrrAHKLotgLA3uOMzfDKMYgiNv0PmcGWz/EH4MtEMRfBXgAdlN0Dg9wjRwXpADkHQEUBQRRFhqrRuG9WkB8QA1cs5TM19I8yO02vIK9NojpegwtvSs1LdWamIr490Z9qpPLSQ7eQuWcWZSllLbph/NGAXBZAEgBIn1IBWbWxskwDRPfpRdSVyg9VeLPxEqWYNB6M/gOao6ZQO6QlXZecYtBQRop8pPEp9QuXN0VLDbrwlujl4YkNQeXqr0Zp4vvxVhKAg/HIEkzslWQJKd0eymIcjNG0a03RUJF4XItgGylokxcfBOY4XLJg7mW+Z7EhtEyPmzdaHGLom64WDrcsX12d6kq4nbaiZC+dZpgkFwQfhZIpy3vlEiPc9LOJtKLC9Lh5bxXLBzyN8crgXXtpYaPhBJyc/YcA+DnpremW+R/JA6zEDyHVPi1N79GMT5Pw1gRt/UTU8n3IKmMlZCrLlIm0qzziOrifZnpqk773NdWDTf8HJB02kNX45u5FZ7wvfX47DS9WmXBTX7PL547+auebmhda2R8yBLt4sb1IpCWIJaG0yY8ia9dEO9LI6s8k69FHfGhg3YbnCWW5ghDpjBjQdRbKekzU231UyooXJLO1FlierqRzWOLe6RqlI+D3zQA6M0umaIQ/NgKSrduxMG5CH5CpXGXN3OQ2G6drFRySOCG0RTRly6aFVcnZNdDFuKH05uYvs7gkELoLW/mR0ej9GLy6EjhuYS+WyAm3VUvoBpfK1x00fIbWqLWy9KbNGmy/QMBvIIDeyRcG6vGHOHueKJG0+vcw5xoeq2I20na9i52Gtwe854M7u6uaEQ65JeDfYEIqIxEC/J+fS+axlE63Firs2x04WajNPd4sWoMDvb5FEtU2S5U3+GF1suUN0hrcznOiw12YDOJijP51NHY0Q49lpBZZLDjuAjN0Rdl6x/Nnz+20n9HzZ/FrRJsxfQVKP9LwOcn5zpDtfS0ga+DmRWBaOuH89jGoZ1e6lbUxU58Q+OouD5JB1JqklGGTunW6Ie94U6DI6SF2eJvloQfTuAHq/nD3pAdBhK/zfTKugomJXA3T6V0v8dhYMc48oK5Im7D759I51M0bpLkRHMgPGLsLJYANPlmKq2ENdLREkUOiTbdJaRMgRpCloPpG0FudvbWSSwnviHoQCqWYFnDSCNI9tEt8gEEPZiF0YIq4PjrtAMUx84cpUJJEfURRu4RiD9t4Dv2XErb3jBu1Y1v3eeAHYGYP+/y4ByByP22m8FxBCJ72s2XT3iwx11utUcgFr/sZks7ApE97eYryCjHnrubq529tDx3lcYId7Is3nN+QY9E79e/fliJ92UvXjXJrEaROAWS3AT4TqEAlfMBn/649uaaNEB2fOMt7fRrIduNPNLLF302yexf0hPz0L1TxO3iEA+kdyAMEHZcBztH9PpboM30Mgq/3IHwkwDCNcbLo1ZLPvil2Xm132T/31qwWS0viLETTNHfnaUnsV2E0ml3ZKndkdoyizku1L5lqXQhxVqzZkkQyZJIV0MCEcFWcxoulglGTezM474XY3iAhbPcjpGPpni7SWvDA93abAvmjbc8XSY9CphOluH7whsQPGFnZ+eifdm8dfwEQasFwszxYyTkMylQKeVW6YKB01Hg7MM6U/smuUI+wjZtnYO5PUUR9mbe1MHIjkJMvfs/1nAMhHrbRTW2u7gkXS5vOVoub2KJsZIYnhLDU+LwlDI8hR14gNSUtbZivIS0kg6cyAtQNIroF0l9L0i+VEpUGGeVlBXJMKbNcuUTRNIrPzzUEfBfIq5TrX+MuN4ZlM+LvWof3kC3jj92pjfOHPVYwzYx+pV10JbgXj6SVtxipAAEMYn8mnYECFcQIhQnPlbEe9ITcJ+EpHkwHW1Uzid4PVPwU7TTkPuvT0yzMMHLBCviNnGIzCOJydJbNqJaSlL2O2ZONtTguiVvBrUWBUEcmeV9a5y4IWA3mAHTinS4v38i3W3K//dP8NmJs1FaBMt6rvWF2W4hY/gKZDhofwtTCl2a8JMl6UyM9OvYikWXqckVegJTclhB7JTfZNtEQdwT2FkL/XtMRWBfz5TJ++FnFHWdmAve+zCm20myJ704ot/6Hl1ukWefEB9dsmD7/iSMIP1Md5cYRMyFgxeDfNhs74MTuCBE8mF7XyAvHT8OwUUzL0Au2fiQvBAGpHLSgrmbtpxi9omwF9NUicOMRyrX9Bae70T+Hd1BddqZoE6n2d5vwghfo+izF6M6Rk2GQrNYAoteKhy4GY7/2bkrNMjggyD8zM0uoroGhL1mO4Xgdq/Zfg4CzGg8ApTF15yf2q4S3tnuJDMvc6YVS9BC080ziR43CYvcKVbNj8IteZOdE3wUhB1aIYLEr00lvMOtz637pKwu5VWc+b2Hr8ME90Jc8tKMMT21RC5sxa2PzVZrvlXaID5x3clVEuCkGXGMC7Grx1fNrXjD/GztfKizPM6I4395yJcSTBmQLOjLsP5RiYL9/Ux28mIq2zuNxks9U0i32BsswusruGjmJD7OLfBi2U3xnkG8arJ8Hp93HveRFEffwcWpW6xsb5QvRd+iQqmvKskiqhHNaMtUGhF2dipKLiN064VJ3P8Tla2VWVK6loJXnhSD2yz3kpbivoz2xa+XJDdydzh1zvmzsi2I97dZDBTs8/vveln1oP2RMquJ8l/ixo+H1eZcffuv8ZKLy8zC64jfbsT4Sfjm2OaLyjPzIPGxp0bT60puTpfqLZw5ssJjNE5IZ5dm6AbA979/oi3E2myYReGCDk3DADukbOuER9wEfQbd8rsL77K5qHI475BOIAgxDYfdTGV87eDd9NSVvXkKs8a611SWlFmsTpGNSe5rCW6DDOoVNXJeksqemsZeotILE9Zzk9Vg0rd0m/5XCZ7m1d+aoZ4tKN8Ipilpczp4FtpZ4D817J+mNrsRLzdgmzfSWQ/GTgWKU4JObf/F7rX+Yt1XulXxZvBY7gX0xYsx233MyY7jGkFB389TWITY3ifMd1kbycsCWRcGf0Tn90ifxWmhvLwsctwEkKKiMtNbVW50JUChE/z4Y16D8j/MJgoJ9GaqfqjCrrH5oOVblsjQ/+ra/px1sWpFvHPVaDiue5Ye9w1DF/WdK1S96+qrx1rfNi1DH55yZ3TpNwseCCJPIMDf/vY3EMTsL8CHo55mUwJTgLdQ/Ml25+2P9d9ksMNHCEIXgU/0gZKEdCMfIce9I1UmRgHeJdX52gvmZK/uuOufj5TbNnav6LpkRp0EHMK6FnGz2RSKA1t6bla7ympQ1BARw9S8Xu2K97wiq5pL0TpuJe25rW6EFuEteol5EQhrOr3Awk+HolUV28rB4JWrEO3+ScpVxT5Juf8f3TLVUo/4/p3WH2uGqY1OGv8XAAD///PxyKelTgAA
+    H4sIAAAAAAAC/9xc/1ejSLb/PX/FHYYZdUaSEG3bsZfewQRtnvn2gNjTz3Y5GCqRkUAaCrtdzfvb36migIIQW9vZebPrnj6L1K17b33u16rC+f671pUXtK6c+LqhGYZtfjAtbdC1+rZpqYZln6h6X9kHOtbtjyY9Wx/qlm3pA200sZRX6ciJ3tfs96rVfZePHKQj70b9nv1e7evDyW/qqTa0lNfpgKH1NdXUaggOUwJ1bNn60LTUfj9n+ks61Bt1zzRjbbTTLg33Ru+H/ZHaK8bl0viZ9qGGplOiIUoQunx4rzTMYdRhIA1Gxx/otL5uFjh1GFAD0z4dn24QfcBxWFsbg607GlqqPtSM3joJA86YDLvrgwy6s0OTEAz14Wk+ttcuxta02mOgnU2ONeIWw5Fln4wmw56yx6DSB6c10/aKaX3N4pHa2y9bcXBqjyecqnuv1pbateroDtbpDJ3ouE66GT3qutyiDtcozzXD1EdDWx+eq329p+wxKLtDfX3d+3Ju6LEx6tk97biGqLNOND47tdVejwUcQ29kqJxJNMMYGco+gyeNVG4hdOJh5kO9sTE61thbpvFoYh2TVZLVDdOhV5zt1bFum5pxrhk8gVxL0Buadn80OpuMGVmnlkz9n4mhrRNnzqFaan0sHLQ5imoQHsjcYNXRD5gi5+96VdserDvWmuDXbS6I1keZ5O7EtEYD29RUo/vO7o0Gqj4007UdMgan48n6/MM9btDQiWfxkXG4vz5cXd/hK45GO9e7mj3uT071YYnRQUGkD09GtjEa2N2RYUzGltZTDlk4mKe/bZTzS3udhpPwi1xk6Z6q9wvr/MLl78m4p1paMZRFjqkRxz/XaVyZXUMfW9ROhqb2Ck5yu80JIel0Mj411J7GUXBqjCfGKXN4ud3Jg4QkBEMjZlDkNsO/q5u2apr66dA2RiPLHr9XZFleG6P+M9aMgW4STRVZZmzHKk1d3dH4AyVSZJnnPB73P9hj1TTfjwwabCf6qSLLlcDNa8iIhYUsZxktTWNr7iPLr0sEo7FmqBbBME0NspxlL8uoGfwFGilYNChNS+2e2aeaZavGwLZGZ9pQkbMSWiUZatb7kXHG1jJJGStyVlGr5ObkeKhZ9tjQTvTfFJkU1Twojyd6P8tlMqmLKSjv1bHdNTTiLikYWVXiR/ShOTk50bu6NqQecWabY7WrKTKpUpTa0vraeGRYvVLaVOQszxXjmcOnw9lCDEs/UbukXhmaOiBVMuOjyFmOqyFizBSZpLa08bGsMXHy3z7YXZXgdq4ZliJniY1FBhnSDMtU5IO8OWFqm1qXgGr1Tft4NLJMy1DHeTnVftO6Wdzn3nGQGbinm8dqX7P1saUe9zVTkUlaS3OmMTEtwmHdubLk1tNNMss2zXeK/DrzeEMfqMYHe6h3bX3MJVX5NfN8fWhqhmXrg55pG5ppERcl3mdMiCZDa2QP1OFpX7OpTor8ev+p8070vqUZ+TwWRj0ChLZh3glJeBV5B0+dV5H3mnNeQyO2sA3tvye6ofWUTpaihiOaFNRTzWTAdLLcZA7VsflulGdDvmlsZ5ExNvRzMkjqJ2kFiFd2snRFXvLu2mnvZ2GTt+sD1TxjTNlY19B62tDS1X6abHu1PXE7i0DSzlRbnU77ABqNohWhTRXRhmn/Gipj5D1BlVR/W1Y67cMqBVd836vmQOm0f3mUyZ7SkduPUuwrnSx3b6B4pXTkDBXWh2ROnvenmnFOAplEo36id4ktjJGVJlBLPVU68l6j4c1g5gUutBCegoTvlghmuz5IgbNAIPwkRchHTowEkJaRF2CQPiUehs7blotuW0Hi+/AA8wgtQfoEW82tN4CvUdAAABiZirgdhxEGKaLsWzk3Msf5fANbCwdPr7fF9i60/rGt9+y+fqYp29MwQmG886D3lO3mTzs7YmsXnB24h1QFHCbLJYq2nYvOJTgXe5c7bwB98TCstnaY5Mzsz9Qgd5ZnC4YHwBFILmwJWzsN5MeIaoKm1yEIFdFBiGEWJoErNGZeozE5ngytiT0y7aE60BQh/V1oDFSDOFUxwF4UI7RbrA6Tl0KDli+6Ay0IindCg+19lFYSRy0/nDo+3TTfJFdoiv1GWsjTUfLeDac3KGqgL0uCJm3UzhVBvC/augzxI2UllOg00+KlBLee6zlMQG4muSnvNWVJzmYOz/Wermas9YF6qtnmOzUTmb/gpdXNIW6eqXm+ksT7DYzXuHALsz6MtcpayauviFaExTRqLrxpFMbhDDen4aLl3MTknzRfJoUqFbYFU9LxpN1umia7g54iTImXBXBzGDe9EKIkAIkE5q3nozlyQZKiBUhSgLB0HcYYJOmzh6+lIIal5x61llE4bcmtIG4tPUK8CBPi1ndLpFx5gbsbR1OlFS5xa75Mdt0YK61FkP4SLrEXBrESEbrHZjpTSljMzl6UOOSLZH6wts7U4eoWGCCssLUtPZc93kKm9lGmcv6SKXDEa0OBEhqkx+6q3Xea3dMNpXXrRK2pM71GLYfMi6bX3i2KWw3SL6tD0qBxxFEYYkKXTmho59rQInvC01PaYmX8/HDecv6ZRKg1yH1BJb83tS8YBTHRptlNYhwuzGnkLXEL3aIAx61Gd2L0yf52PLGUFl4sW9Mk8u1bFF2FMWqGCW7Q2sBThJETr1MY2qluWsYH1uqm7T3NSYS+NQ2DmTdv3jkLHxqNCOHobrpwbW9mzxzPTyK0vQP3NJuRMQ/Fiii/gc+Oh+3YR2ipiJ03gL0FChOsiHtvIL72Zhh+/LHyQFnMwgg88AIQt2P0CWQQGdOdN+CGlIT8MHYgZg+CeP/rSiC8riLk3MDDA3zMqb0ZXIDogYQ+5fzgkitF2Q9Nx9oXNE0wcuGjIP76USATiZT4TYk0QjiJApDzl3lOz37o4kEsgMhHZx59dMOAqwKbxa4K0IPYD8ObZJlDzsMsczATyEPs+DYPfISmYeQq4j6dGmMnwnRcEbddByP4+Yc4LZCfrz0fAY4SVEI9kw5SzjXDX6IqKm2CLxGywRLTJIpQsEFqYartbZ4QJE7XnR2Q5gjE0uo2G1OwCB3nL6VpESLB6e4+a2XE65ErPO4NzMSb/SA3/qOQpGvIXeNZaobR14CM0TQM3FgouVgR13YQ2jF2cPxXCHAS3y8N7z8qZjm05gjb2ImuHN/PUcJOZD+GVEpOkUoiPwvH1NTiPTd7lQEubICOo63A50Qg4X/OKAmR9hToOG7fCF9mrIM2cWsfpFls9m9BpM9hocxbELnqBZ23P8olPkStCxD/Dt8p0IbLOmVo6Di4xKhEwEXgSy1rz6JwYUdo7sU4urNJy2ST0vhCi5NZs9B3UUTi3ovSLVUGkrDzn+gWBDZYkm0h7xQMBtJ1MZCltOsA8X5ji7IibsS1OC9zI47RH+5GV17gRHdf9SJG9ogjMYqlg6+rzpTJWPcnblLFp8oCK261wbfKc8ruRd1GmlVk1joNdbrNHlNywLJI2GTGNT+s5fx1fywh+Ue75DPc8quuWeOej7toZcIGj6XbiJnnF309ffPE2k8mpv65z/zzVcnneF5fTWQ8cdnVLqijiZk0uLzMU1l9FuNZvTCN5T1JbY3LVSK2hwfACJWrXXE09lepeZRgFkZlqz9i8MLIe5R06dwil+3v6V526oeJK3mBhyVCGzcpBaVNDwNPPhFnyPisBBApBTEiM0T7Oe1iznTre210ssVZ4XGv+DO6RUqJXJA8EFpEvZYrFAoWwVHo/PYtg6OxasR3MUaLKSbOS5v35zXjEN9OSR3IysS3tt+5GuA6aBEGUoT80HGfMoHpDSJT5U/o4nPZZPuSxIVoSQpCaenMSWIvh9/vYRIFjk8mSUk+4ZvCqTBZjMPlv529iNJ/prGejarrxc6Vj/7tgGV6/zWxTTMMWcrzcH0pmDTgpBSkvw4gtyiKvTCw5zhzM4xiDIK4TW96ZrD1Q/wx2AJB/FWAB2B3SOfwANfIcUEKQN4RQFFAEGWhsWoU3qsFxAfUwDVLyXwtzYPcbsMr2GuDmK7H0NJbVNNSrYmpiH9v1Kc6uZzk4C1UzplFWUppm344bxQAlwWAFCDSh1RgZm2cDNMw8V16VXWF0lMl/kysZAkGrTeD76DmmAnkDllp5xW3GBSkkSI/SXxK7cLVXcFisy68NXppSFJzcKnam3G6+F6MpSTwcAySNCNbBUlySveagig3YxTdelMkVBQu1wLIVirKxMU3gRkulzyYa5nvSWwYLePD1o0Wtyjqhoulwx3bZ7eaqiJup50I6VunCQbJBeFngXTa8k6J9Dgn7WwivbggHV7Oe8XCIX9zvBJY115q+EgoITdnzzEAfm56n7pF/kfiMAvBc0iFX3vzaxTj8zSMFXFbPzGVfA+SylgJueoiZSLNOo+oLt6Xma7qtM99bdVww88BSac9dDW+mVvhCd9bj89O00tXFtzk9/xKupO/6umG1rVGxocs0S5uXC8CaQliaThtwpP42gXxvjSyyjP5WtQRHzpot8FZYmmOMGQKMxZEvZWSPjPVVj+lgsIl6UydJaanG9k8trhHqkb5OPhNA4De+ZIpCsGPraB060YcnIvgJ1Qad3kzB4nt1slKJYcEbhhNEX3pollxdUJ2PWQhfji9ienrDA4phN7yZn50NEovJo+OFJ5L6LsFYtJd9QKq8bXCRRctv6Elar0svUmTJts/EMArOLBHwrWxaswR7o4najS9zj3MiabXiridpG3vYqfB7THvyeDu7opGpEN+OdgXiIDKSLQg79f3omkcpcONtTrLRhduNkpzjxerxuBgn0+xRJXtQvUdXmi9THmDtDaX47zYYAc2k6g4k08djR3t0GMJmUUGO46L0Bx9Ubb+0fz5Yyv9d9T8WdwqwVZMX4HyvwR8fnKuM1RLTxv4OphZEYi2fjiPbRza6aVuRV3sxDc0jorrk3QgpSYZZeiUbo1+2BvuNDhCWpgt/mZJ+OEEfrCaP+wN2WEg8dtMr6yrYFICd/NUSvd7HAZ2jCMvmCviNvz+iXQ+ReMmSU40B8Ijxs5iCUCTb6bSSlgjHS1R5JBo011CyhSoIWQ5mL4R5GZnb53EcuIbgg6kYgmWNYw0gmQf3SIfQNCDWRgtqAKOv047QHHszFEqlBRRH2HkHoH40wa+Y8+ltO0N41bd+NZ9DtgRiPnzLg/OEYjcb7sZHEcgsqfdfPmEB3vc5VZ7BGLxy262tCMQ2dNuvoKMcuy5u7na2UvLc1dpjHAny+I95xf0SPR+/euHlXhf9uJVk8xqFIlTIMlNgO8UClA5H/Dpj2tvrkkDZMc33tJOvyOy3cgjvXzRZ5PM/iU9MQ/dO0XcLg7xQHoHwgBhx3Wwc0SvvwXaTC+j8MsdCD8JIFxjvDxqteSDX5qdV/tN9v+tBZvV8oIYO8EU/d1ZehLbRSiddkeW2h2pLbOY40LtW5ZKF1KsNWuWBJEsiXQ1JBARbDWn4WKZYNTEzjzuezGGB1g4y+0Y+WiKt5u0NjzQrc22YN54y9Nl0qOA6WQZvi+8AcETdnZ2LtqXzVvHTxC0WiDMHD9GQj6TApVSbpUuGDgdBc4+rDO1b5Ir5CNs09Y5mNtTFGFv5k0djOwoxNS7/2MNx0Cot11UY7uLS9Ll8paj5fImlhgrieEpMTwlDk8pw1PYgQdITVlrK8ZLSCvpwIm8AEWjiH6R1PeC5EulRIVxVklZkQxj2ixXPk4kvfLDQx0B/43iOtX6Z4rrnUH5vNir9uENdOv4Y2d648xRjzVsE6NfWQdtCe7lI2nFLUYKQBCTyK9pR4BwBSFCceJjRbwnPQH3SUiaB9PRRuV8gtczBT9FOw25//rENAsTvEywIm4Th8g8kpgsvWUjqqUkZb9j5mRDDa5b8mZQa1EQxJFZ3rfGiRsCdoMZMK1Ih/v7J9Ldpvx//wSfnTgbpUWwrOdaX5jtFjKGr0CGg/a3MKXQpQk/WZLOxEi/m61YdJmaXKEnMCWHFcRO+U22TRTEPYGdtdC/1FQE9vVMmbwffkZR14m54L0PY7qdJHvSiyP6FfDR5RZ59gnx0SULtu9PwgjSD3h3iUHEXDh4MciHzfY+OIELQiQftvcF8tLx4xBcNPMC5JKND8kLYUAqJy2Yu2nLKWYfD3sxTZU4zHikck1v4flO5N/RHVSnnQnqdJrt/SaM8DWKPnsxqmPUZCg0iyWw6KXCgZvh+J+du0KDDD4Iws/c7CKqa0DYa7ZTCG73mu3nIMCMxiNAWXzN+antKuGd7U4y8zJnWrEELTTdPJPocZOwyJ1i1fwo3JI32TnBR0HYoRUiSPzaVMI73Prcuk/K6lJexZnfe/g6THAvxCUvzRjTU0vkwlbc+thsteZbpQ3iE9edXCUBTpoRx7gQu3p81dyKN8zP1s6HOsvjjDj+l4d8KcGUAcmCvgzrH5Uo2F/WZCcvprK902i81DOFdIu9wSK8voKLZk7i49wCL5bdFO8ZxKsmy+fxeedxH0lx9B1cnLrFyvZG+VL0LSqU+qqSLKIa0Yy2TKURYWenouQyQrdemMT9P1HZWpklpWspeOVJMbjNci9pKe7LaF/8eklyI3eHU+ecPyvbgnh/m8VAwT6//66XVQ/aHymzmij/JW78eFhtztW3/xovubjMLLyO+O1GjJ+Eb45tvqg8Mw8SH3tqNL2u5OZ0qd7CmSMrPEbjhHR2aYZuAHz/+yfaQqzNhlkULujQNAywQ8q2TnjETdBn0C2/u/Aum4sqh/MO6QSCENNw2M1UxtcO3k1PXdmbpzBrrHtNZUmZxeoU2ZjkvpbgNsigXlEj5yWp7Klp7CUqvTBhPTdZDSZ9S7fpf6/gaV79rRnq2YLyjWCakjang2ehnQX+U8P+aWqzG/FyA7Z5I531YOxUoDgl6NT2X+xe6y/WfaVbFW8Gj+VeQF+8GLPdx5zsOK4RFPT9PIVFiO19wnyXtZG8LJB1YfBHdH6P9FmcFsrLyyLHTQApKiozvVXlRlcCFDrBjz/mNSj/k22ikEBvpuqHKuwamw9avmWJDP2vru3PWRerVsQ7V42G47pn6XHfMHRR37lC1buuvnqs9W3TMvThKXdGl36z4IEg8gQC/O1vfwNBzP42fDjqaTYlMAV4C8Ufc3fe/lj/TQY7fIQgdBH4RB8oSUg38hFy3DtSZWIU4F1Sna+9YE726o67/vlIuW1j94quS2bUScAhrGsRN5tNoTiwpedmtausBkUNETFMzevVrnjPK7KquRSt41bSntvqRmgR3qKXmBeBsKbTCyz8dChaVbGtHAxeuQrR7p+kXFXsk5T7/9EtUy31iO/faf2xZpja6KTxfwEAAP//TkMp6r9OAAA=
 
 
 

pkg/agent/testdata/AKSUbuntu1604+Containerd/line9.sh

@@ -133,8 +133,8 @@ export GPU_DEST=/usr/local/nvidia
 DOCKER_VERSION=1.13.1-1
 export NVIDIA_DRIVER_IMAGE_SHA="${GPU_IMAGE_SHA:=}"
 export NVIDIA_DRIVER_IMAGE_TAG="${GPU_DV}-${NVIDIA_DRIVER_IMAGE_SHA}"
-export NVIDIA_DRIVER_IMAGE="mcr.microsoft.com/aks/aks-gpu"
 export NVIDIA_GPU_DRIVER_TYPE="${GPU_DRIVER_TYPE:=}"
+export NVIDIA_DRIVER_IMAGE="mcr.microsoft.com/aks/aks-gpu-${NVIDIA_GPU_DRIVER_TYPE}"
 export CTR_GPU_INSTALL_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind"
 export DOCKER_GPU_INSTALL_CMD="docker run --privileged --net=host --pid=host -v /opt/gpu:/mnt/gpu -v /opt/actions:/mnt/actions --rm"
 APT_CACHE_DIR=/var/cache/apt/archives/