From 1e162d3ba78306d538be32c8a8c931f603ceec4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B0=B1=EC=A7=80=EB=AA=85?= Date: Fri, 4 Sep 2026 11:29:37 +0900 Subject: [PATCH 1/2] nvidia_dcgm: do not replace the DCGM that DGX OS already ships MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On DGX systems install-dgx.yml unconditionally installs the unversioned datacenter-gpu-manager package, which the CUDA repo resolves to DCGM 3.3.9. DGX OS 7.x ships datacenter-gpu-manager-4-cuda13 (4.5.2). The two series conflict, so apt removes the 4.x packages and installs 3.3.9. The play reports ok and nothing stops; the regression only shows up later: dcgmi diag fails with "Detected unsupported Cuda version" (3.3.9 has no CUDA 13 plugin) and dcgmi discovery -l reports 0 NvSwitches on a DGX B300 that has two. The Ubuntu branch already honours dcgm_pkg_name; only the DGX branch hardcodes the name. roles/nvidia-dgx/vars/ubuntu-24.04.yml lists datacenter-gpu-manager-4-cuda13 for DGX OS 7, so the two roles disagree. Skip the install on DGX when any datacenter-gpu-manager package is already present, and use dcgm_pkg_name otherwise. Observed on DGX B300, DGX OS 7.5.0, driver 580.126.20. Signed-off-by: 백지명 --- roles/nvidia_dcgm/tasks/install-dgx.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/nvidia_dcgm/tasks/install-dgx.yml b/roles/nvidia_dcgm/tasks/install-dgx.yml index 0d512b3e7..87442aba5 100644 --- a/roles/nvidia_dcgm/tasks/install-dgx.yml +++ b/roles/nvidia_dcgm/tasks/install-dgx.yml @@ -1,5 +1,12 @@ --- +# DGX OS ships its own DCGM (4.x on DGX OS 7). Never replace it with the +# unversioned 3.x metapackage; only install when nothing is present. +- name: gather package facts + package_facts: + manager: auto + - name: install DCGM from repos package: - name: "datacenter-gpu-manager" + name: "{{ dcgm_pkg_name }}" state: present + when: ansible_facts.packages | select('match', '^datacenter-gpu-manager') | list | length == 0 From b348f62bf4db15fd3b6d4c64fbde68ee85c91926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B0=B1=EC=A7=80=EB=AA=85?= Date: Sun, 6 Sep 2026 12:32:09 +0900 Subject: [PATCH 2/2] nvidia_dcgm: converge DGX hosts on the DCGM package their DGX OS ships MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checking for any datacenter-gpu-manager* package was too loose: a host already downgraded to 3.3.9 stayed there, a partial 4.x install was accepted, and the absent case fell back to the unversioned metapackage that caused the downgrade in the first place. Map each DGX OS release to the package it ships (taken from roles/nvidia-dgx/vars) as dcgm_dgx_pkg_map, overridable through dcgm_dgx_pkg_name. On DGX: fail on an unmapped release, remove installed packages from the other DCGM series, then install the mapped package when it is missing. A host already on the right package is left alone, so a second run reports ok. Signed-off-by: 백지명 --- roles/nvidia_dcgm/defaults/main.yml | 9 +++++++ roles/nvidia_dcgm/tasks/install-dgx.yml | 33 ++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/roles/nvidia_dcgm/defaults/main.yml b/roles/nvidia_dcgm/defaults/main.yml index 087cf9c0c..7b7e24b98 100644 --- a/roles/nvidia_dcgm/defaults/main.yml +++ b/roles/nvidia_dcgm/defaults/main.yml @@ -1,6 +1,15 @@ --- dcgm_pkg_name: "datacenter-gpu-manager" +# DGX OS ships its own DCGM and the package name differs per release. +# install-dgx.yml converges on this package and removes the other series. +dcgm_dgx_pkg_map: + "18.04": "datacenter-gpu-manager" # DGX OS 4 + "20.04": "datacenter-gpu-manager" # DGX OS 5 + "22.04": "datacenter-gpu-manager" # DGX OS 6 + "24.04": "datacenter-gpu-manager-4-cuda13" # DGX OS 7 +dcgm_dgx_pkg_name: "{{ dcgm_dgx_pkg_map[ansible_distribution_version] | default('') }}" + # RedHat family epel_package: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" epel_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" diff --git a/roles/nvidia_dcgm/tasks/install-dgx.yml b/roles/nvidia_dcgm/tasks/install-dgx.yml index 87442aba5..74d09a794 100644 --- a/roles/nvidia_dcgm/tasks/install-dgx.yml +++ b/roles/nvidia_dcgm/tasks/install-dgx.yml @@ -1,12 +1,37 @@ --- # DGX OS ships its own DCGM (4.x on DGX OS 7). Never replace it with the -# unversioned 3.x metapackage; only install when nothing is present. +# unversioned 3.x metapackage from the CUDA repo; converge on the package +# DGX OS expects and remove packages from the other series first. - name: gather package facts package_facts: manager: auto -- name: install DCGM from repos +- name: fail when this DGX OS release has no known DCGM package + fail: + msg: >- + No DCGM package is mapped for {{ ansible_distribution }} + {{ ansible_distribution_version }} on DGX; set dcgm_dgx_pkg_name to + the package shipped by this DGX OS release. + when: dcgm_dgx_pkg_name | length == 0 + +- name: collect installed DCGM packages from the other series + set_fact: + dcgm_dgx_conflicting_pkgs: >- + {{ (_dcgm_installed | reject('match', _dcgm_v4_re) | list) + if (dcgm_dgx_pkg_name is match(_dcgm_v4_re)) + else (_dcgm_installed | select('match', _dcgm_v4_re) | list) }} + vars: + _dcgm_installed: "{{ ansible_facts.packages | select('match', '^datacenter-gpu-manager') | list }}" + _dcgm_v4_re: '^datacenter-gpu-manager-4(-|$)' + +- name: remove DCGM packages from the other series + package: + name: "{{ dcgm_dgx_conflicting_pkgs }}" + state: absent + when: dcgm_dgx_conflicting_pkgs | length > 0 + +- name: install the DCGM package shipped by DGX OS package: - name: "{{ dcgm_pkg_name }}" + name: "{{ dcgm_dgx_pkg_name }}" state: present - when: ansible_facts.packages | select('match', '^datacenter-gpu-manager') | list | length == 0 + when: dcgm_dgx_pkg_name not in ansible_facts.packages