diff --git a/config.example/group_vars/all.yml b/config.example/group_vars/all.yml index 3ab9fa6e2..7c433e23d 100644 --- a/config.example/group_vars/all.yml +++ b/config.example/group_vars/all.yml @@ -199,19 +199,6 @@ enroot_environ_config_files_dgx: content: | OMPI_MCA_btl_tcp_if_exclude=lo,docker0,ib0,ib1,ib2,ib3 -# Singularity configuration -# Playbook: singularity, slurm-cluster -# Singularity target version -# set an alternate singularity version here; -# see roles/singularity_wrapper/defaults/main.yml for default -# singularity_version: -singularity_conf_path: "/etc/singularity/singularity.conf" -bind_paths: [] - # example: - #- /mnt/shared -golang_install_dir: '/opt/go/{{ golang_version }}' -golang_gopath: /opt/go/packages - ################################################################################ # AUTH # ################################################################################ diff --git a/config.example/group_vars/slurm-cluster.yml b/config.example/group_vars/slurm-cluster.yml index 688517fc8..89bccdde4 100644 --- a/config.example/group_vars/slurm-cluster.yml +++ b/config.example/group_vars/slurm-cluster.yml @@ -66,7 +66,6 @@ dns_late: false slurm_configure_etc_hosts: yes slurm_cluster_install_cuda: yes slurm_cluster_install_nvidia_driver: yes -slurm_cluster_install_singularity: no ################################################################################ # NFS # @@ -164,7 +163,10 @@ openmpi_configure: "./configure --prefix={{ openmpi_install_prefix }} --disable- # Open OnDemand # ################################################################################ install_open_ondemand: no -# OOD Linux-host adapter requires `slurm_cluster_install_singularity` to be true +# The OOD Linux-host adapter runs interactive apps inside an Apptainer/Singularity +# container. DeepOps no longer installs Singularity (the wrapper role is retired; +# Enroot/Pyxis is the supported Slurm runtime), so install Apptainer/Singularity on +# the login nodes yourself before enabling this adapter. ood_install_linuxhost_adapter: no servername: '{{ ansible_fqdn }}' diff --git a/config.example/host_vars/gpu01 b/config.example/host_vars/gpu01 index 331c583ff..8275774be 100644 --- a/config.example/host_vars/gpu01 +++ b/config.example/host_vars/gpu01 @@ -2,5 +2,5 @@ # Example of a host-specific variable file # These variables would only be used for a host named "gpu01" -# Don't install Singularity on this host -#slurm_cluster_install_singularity: false +# Don't install the CUDA toolkit on this host +#slurm_cluster_install_cuda: false diff --git a/docs/container/docker-login.md b/docs/container/docker-login.md index 48eb3017c..f33a3e791 100644 --- a/docs/container/docker-login.md +++ b/docs/container/docker-login.md @@ -43,11 +43,11 @@ The [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-co ## Slurm jobs using private registries -The process for using private registries is different depending on whether you are using Singularity or Enroot as your container runtime. +The process for using private registries depends on your container runtime. Enroot/Pyxis is the runtime DeepOps installs and validates for Slurm; the Singularity/Apptainer notes below apply only if your site provides that runtime separately. ### Singularity -[Singularity](https://sylabs.io/singularity/) gets container pull credentials using environment variables: +DeepOps no longer installs Singularity (the wrapper role is retired; Singularity lives on upstream as [Apptainer](https://apptainer.org/), and Enroot/Pyxis is the supported Slurm runtime). If your site provides Apptainer/Singularity, it gets container pull credentials using environment variables: ```bash export SINGULARITY_DOCKER_USERNAME= diff --git a/docs/deepops/architecture.md b/docs/deepops/architecture.md index 1e5956edb..498292843 100644 --- a/docs/deepops/architecture.md +++ b/docs/deepops/architecture.md @@ -109,7 +109,6 @@ Roles run as part of the various playbooks. Includes default values and tasks as │ ├── roce_backend │ ├── rsyslog_client │ ├── rsyslog_server -│ ├── singularity_wrapper │ ├── slurm │ ├── spack │ └── standalone-container-registry diff --git a/docs/slurm-cluster/README.md b/docs/slurm-cluster/README.md index 5d60986c4..0fa9d534a 100644 --- a/docs/slurm-cluster/README.md +++ b/docs/slurm-cluster/README.md @@ -14,7 +14,7 @@ Instructions for deploying a GPU cluster with Slurm - [Configuring shared filesystems](#configuring-shared-filesystems) - [Installing tools and applications](#installing-tools-and-applications) - [Installing Open on Demand](#installing-open-on-demand) - - [Pyxis, Enroot, and Singularity](#pyxis-enroot-and-singularity) + - [Pyxis and Enroot](#pyxis-and-enroot) - [Large deployments](#large-deployments) ## Requirements @@ -236,9 +236,11 @@ See the documentation on [software modules](./software-modules.md) for informati [Open OnDemand](https://openondemand.org/) can be installed by setting the `install_open_ondemand` variable to yes before running the `slurm-cluster.yml` playbook. -## Pyxis, Enroot, and Singularity +## Pyxis and Enroot -[Pyxis](https://github.com/NVIDIA/pyxis) and [Enroot](https://github.com/NVIDIA/enroot) are installed by default and can be disabled by setting `slurm_install_enroot` and `slurm_install_pyxis` to no. Singularity can be installed by setting the `slurm_cluster_install_singularity` variable to yes before running the `slurm-cluster.yml` playbook. +[Pyxis](https://github.com/NVIDIA/pyxis) and [Enroot](https://github.com/NVIDIA/enroot) are installed by default and can be disabled by setting `slurm_install_enroot` and `slurm_install_pyxis` to no. They are the supported, release-validated container runtime for Slurm in DeepOps. + +The DeepOps Singularity wrapper role has been retired. Singularity lives on upstream as [Apptainer](https://apptainer.org/); sites that still want it can install Apptainer/Singularity separately. ## Large deployments diff --git a/docs/slurm-cluster/slurm-single-node.md b/docs/slurm-cluster/slurm-single-node.md index 94518be98..f966d6d31 100644 --- a/docs/slurm-cluster/slurm-single-node.md +++ b/docs/slurm-cluster/slurm-single-node.md @@ -11,7 +11,7 @@ Single Node Slurm Deployment Guide - [Allocating GPUs](#allocating-gpus) - [Running Containers](#running-containers) - [Rootless Docker](#rootless-docker) - - [Enroot and Singularity](#enroot-and-singularity) + - [Enroot and Pyxis](#enroot-and-pyxis) ## Introduction @@ -92,7 +92,7 @@ The general requirements and procedure for Slurm setup via deepops is documented b. Configuring `"config/group_vars/slurm-cluster.yml"` - Typically users cannot ssh as local users directly to compute nodes in a Slurm cluster without a Slurm reservation. However, since in this deployment a compute node also functions as a login node we need to add users to slurm configuration for the ability to ssh as local users. Again, this is needed when a compute node also functions as a login node. Additionally, set the singularity install option (which is no by default). Singularity can be used to run containers and it will be used to set up rootless options as well. Do not set a default NFS with single node deployment. + Typically users cannot ssh as local users directly to compute nodes in a Slurm cluster without a Slurm reservation. However, since in this deployment a compute node also functions as a login node we need to add users to slurm configuration for the ability to ssh as local users. Again, this is needed when a compute node also functions as a login node. Do not set a default NFS with single node deployment. ```bash vi config/group_vars/slurm-cluster.yml @@ -101,7 +101,6 @@ The general requirements and procedure for Slurm setup via deepops is documented ```yaml slurm_enable_nfs_server: false slurm_enable_nfs_client_nodes: false - slurm_cluster_install_singularity: yes slurm_login_on_compute: true @@ -331,8 +330,8 @@ attach to the tmux session in the adopted ssh session. ### Running Containers -DeepOps enables running containers with several containerization platforms: -docker, singularity, and enroot with pyxis. +DeepOps enables running containers on Slurm with rootless Docker and with +Enroot/Pyxis, which is installed by default. #### Rootless Docker @@ -441,7 +440,7 @@ cat: /slurm/slurmdbd.conf: Permission denied Rootless docker supports building containers and many standard docker features with a few limitations. These limitations can make it challenging to run multi node Slurm jobs, therefore for multi node jobs on Slurm the recommended -approach is via enroot or singularity. +approach is via Enroot/Pyxis. A user can explicitly stop the rootless docker daemon with “stop_rootless_docker.sh” script, or just exit the Slurm session. Upon exit from a slurm session the @@ -458,11 +457,11 @@ login-session:$ These scripts “start_rootless_docker.sh” and “stop_rootless_docker.sh” appear on a user's path upon loading the rootless docker module. -### Enroot and Singularity +### Enroot and Pyxis -Singularity and enroot could also be deployed via DeepOps. These would be -useful for multi-node jobs if running on more than one DGX system. -Enroot with pyxis can be tested by running: +Enroot with Pyxis is installed by DeepOps by default and is the supported +container runtime for multi-node jobs across more than one DGX system. +It can be tested by running: The examples below use `registry.example.com/hpc/nccl-tests:latest` as a placeholder for a site-maintained NCCL tests image. @@ -504,25 +503,10 @@ will typically be inferred from the environment. Calling mpirun approach could be useful, because certain binding options are not available to srun directly, but can be set via mpirun. -Singularity could be used in a similar fashion to enroot. Don’t forget the -“--nv” option for GPUs. - -```bash -login-session:srun --mpi=pmi2 --ntasks=2 --gpus-per-task=1 \ - singularity exec --nv docker://registry.example.com/hpc/nccl-tests:latest \ - all_reduce_perf -b 1M -e 4G -f 2 -g 1 -``` - -Similarly to invoke mpirun with singularity run script (same script as was used -with enroot): - -```bash -login-session:srun --ntasks=2 --gpus-per-task=1 \ - singularity exec --nv docker://registry.example.com/hpc/nccl-tests:latest \ - ${PWD}/test_allreduce.sh -``` - -Refer to singularity documentation for further details. Building containers with -singularity is permitted to non-privileged users via the “--fakeroot” option. -Enroot and singularity excel at running containerized multi node jobs, which is -somewhat difficult and less convenient to do using docker on Slurm. +The DeepOps Singularity wrapper role has been retired, so DeepOps no longer +installs Singularity. Singularity lives on upstream as +[Apptainer](https://apptainer.org/); if your site installs Apptainer/Singularity +separately, it can run the same containers with `singularity exec --nv` (remember +the “--nv” option for GPUs). Enroot/Pyxis is the supported path and excels at +running containerized multi node jobs, which is somewhat difficult and less +convenient to do using docker on Slurm. diff --git a/docs/slurm-cluster/slurm-usage.md b/docs/slurm-cluster/slurm-usage.md index 41cdd8da9..9ecd21071 100644 --- a/docs/slurm-cluster/slurm-usage.md +++ b/docs/slurm-cluster/slurm-usage.md @@ -128,13 +128,14 @@ scancel JOBID To run a deep learning job with multiple processes, use MPI: ```bash -srun -p PARTITION --pty /bin/bash -singularity pull docker://nvcr.io/nvidia/tensorflow:19.05-py3 -singularity run docker://nvcr.io/nvidia/tensorflow:19.05-py3 -cd /opt/tensorflow/nvidia-examples/cnn/ -mpiexec --allow-run-as-root -np 2 python resnet.py --layers=50 --batch_size=32 --precision=fp16 --num_iter=50 +srun -p PARTITION --gpus-per-task=1 \ + --container-image=nvcr.io/nvidia/tensorflow:19.05-py3 \ + bash -c "cd /opt/tensorflow/nvidia-examples/cnn/ && \ + mpiexec --allow-run-as-root -np 2 python resnet.py --layers=50 --batch_size=32 --precision=fp16 --num_iter=50" ``` +The `--container-image` flag runs the job inside the image using Pyxis/Enroot, which DeepOps installs by default. + ## Additional Resources - [SchedMD Slurm Quickstart Guide](https://slurm.schedmd.com/quickstart.html) diff --git a/playbooks/container/singularity.yml b/playbooks/container/singularity.yml deleted file mode 100644 index 74208898f..000000000 --- a/playbooks/container/singularity.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- hosts: all - become: yes - roles: - - singularity_wrapper diff --git a/playbooks/slurm-cluster.yml b/playbooks/slurm-cluster.yml index 979f20afa..e19a038f2 100644 --- a/playbooks/slurm-cluster.yml +++ b/playbooks/slurm-cluster.yml @@ -137,10 +137,6 @@ hostlist: "{{ rsyslog_client_group | default('slurm-node') }}" when: slurm_enable_rsyslog_client|default(true) -- name: Install Singularity - import_playbook: container/singularity.yml - when: slurm_cluster_install_singularity|default(true) - - name: Install Open OnDemand import_playbook: slurm-cluster/open-ondemand.yml when: install_open_ondemand diff --git a/roles/requirements.yml b/roles/requirements.yml index 897cdda01..3d541e0ba 100644 --- a/roles/requirements.yml +++ b/roles/requirements.yml @@ -32,6 +32,3 @@ roles: - src: https://github.com/OSC/ood-ansible.git version: 'v3.0.3' - -- src: gantsign.golang - version: 3.5.0 diff --git a/roles/singularity_wrapper/.yamllint b/roles/singularity_wrapper/.yamllint deleted file mode 100644 index 882767605..000000000 --- a/roles/singularity_wrapper/.yamllint +++ /dev/null @@ -1,33 +0,0 @@ ---- -# Based on ansible-lint config -extends: default - -rules: - braces: - max-spaces-inside: 1 - level: error - brackets: - max-spaces-inside: 1 - level: error - colons: - max-spaces-after: -1 - level: error - commas: - max-spaces-after: -1 - level: error - comments: disable - comments-indentation: disable - document-start: disable - empty-lines: - max: 3 - level: error - hyphens: - level: error - indentation: disable - key-duplicates: enable - line-length: disable - new-line-at-end-of-file: disable - new-lines: - type: unix - trailing-spaces: disable - truthy: disable diff --git a/roles/singularity_wrapper/defaults/main.yml b/roles/singularity_wrapper/defaults/main.yml deleted file mode 100644 index 08ee8d83b..000000000 --- a/roles/singularity_wrapper/defaults/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -# vars for lecorguille.singularity -singularity_version: "3.11.4" -singularity_conf_path: "/etc/singularity/singularity.conf" -bind_paths: [] - -# vars for gantsign.golang -golang_version: "1.14.4" -golang_install_dir: "/opt/go/{{ golang_version }}" -golang_gopath: "/opt/go/packages" diff --git a/roles/singularity_wrapper/meta/main.yml b/roles/singularity_wrapper/meta/main.yml deleted file mode 100644 index 9fbd94944..000000000 --- a/roles/singularity_wrapper/meta/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -galaxy_info: - role_name: singularity_wrapper - namespace: deepops - author: DeepOps Team - company: NVIDIA - description: Wrap lecourguille.singularity role - license: 3-Clause BSD - min_ansible_version: 2.9 diff --git a/roles/singularity_wrapper/molecule/default/converge.yml b/roles/singularity_wrapper/molecule/default/converge.yml deleted file mode 100644 index c0295f669..000000000 --- a/roles/singularity_wrapper/molecule/default/converge.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Converge - hosts: all - tasks: - - name: "Include singularity_wrapper" - include_role: - name: "singularity_wrapper" diff --git a/roles/singularity_wrapper/molecule/default/molecule.yml b/roles/singularity_wrapper/molecule/default/molecule.yml deleted file mode 100644 index 9689eabd0..000000000 --- a/roles/singularity_wrapper/molecule/default/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -dependency: - name: galaxy - options: - requirements-file: requirements.yml -driver: - name: docker -platforms: - - name: singularity-ubuntu-2204 - image: geerlingguy/docker-ubuntu2204-ansible - cgroupns_mode: host - pre_build_image: true - privileged: true -provisioner: - name: ansible - ansible_args: - - -vv -verifier: - name: ansible diff --git a/roles/singularity_wrapper/molecule/default/verify.yml b/roles/singularity_wrapper/molecule/default/verify.yml deleted file mode 100644 index b5afb1c0d..000000000 --- a/roles/singularity_wrapper/molecule/default/verify.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: verify - hosts: all - tasks: - - name: check for path to singularity - command: which singularity - register: which_singularity - changed_when: which_singularity.rc != 0 - - - name: verify path to singularity - assert: - that: - - "'/usr/local/bin/singularity' in which_singularity.stdout" diff --git a/roles/singularity_wrapper/requirements.yml b/roles/singularity_wrapper/requirements.yml deleted file mode 100644 index 0196599d1..000000000 --- a/roles/singularity_wrapper/requirements.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -roles: -- name: Setup singularity roles - include_tasks: roles.yml diff --git a/roles/singularity_wrapper/roles.yml b/roles/singularity_wrapper/roles.yml deleted file mode 100644 index ee6063e08..000000000 --- a/roles/singularity_wrapper/roles.yml +++ /dev/null @@ -1,4 +0,0 @@ -- src: abims_sbr.singularity - version: 3.7.1-1 -- src: gantsign.golang - version: 2.4.0 diff --git a/roles/singularity_wrapper/tasks/main.yml b/roles/singularity_wrapper/tasks/main.yml deleted file mode 100644 index f4e8890f3..000000000 --- a/roles/singularity_wrapper/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- name: legacy CentOS 8 - ensure powertools installed - block: - - name: ensure prereq packages installed - yum: - name: "dnf-plugins-core" - state: "present" - - name: enable powertools - command: "yum config-manager --set-enabled powertools" # noqa command-instead-of-module - register: enable_powertools - changed_when: enable_powertools.rc != 0 - when: (ansible_distribution == "CentOS") and (ansible_distribution_major_version == "8") - -- name: rhel 8 - ensure CRB repository is enabled - rhsm_repository: - name: "codeready-builder-for-rhel-8-x86_64-rpms" - when: (ansible_distribution == "Red Hat Enterprise Linux") and (ansible_distribution_major_version == "8") - -- name: debian - ensure apt cache is up to date - apt: - update_cache: yes - when: ansible_os_family == "Debian" - changed_when: false - -- name: create a folder for go - file: - path: "{{ golang_install_dir }}" - recurse: yes - -- name: install golang explicitly - include_role: - name: gantsign.golang - -- name: install singularity - include_role: - name: abims_sbr.singularity diff --git a/roles/slurm/templates/etc/slurm/epilog.d/95-lastuserjob-rootless b/roles/slurm/templates/etc/slurm/epilog.d/95-lastuserjob-rootless deleted file mode 100755 index b6bd0cbcf..000000000 --- a/roles/slurm/templates/etc/slurm/epilog.d/95-lastuserjob-rootless +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -command -v singularity || exit 0 -/usr/local/bin/singularity config fakeroot -r "${SLURM_JOB_USER}" || true diff --git a/roles/slurm/templates/etc/slurm/prolog.d/95-all-rootless b/roles/slurm/templates/etc/slurm/prolog.d/95-all-rootless deleted file mode 100755 index 8f972ba8a..000000000 --- a/roles/slurm/templates/etc/slurm/prolog.d/95-all-rootless +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -command -v singularity || exit 0 -/usr/local/bin/singularity config fakeroot -a "${SLURM_JOB_USER}" diff --git a/virtual/vars_files/virt_large_slurm.yml b/virtual/vars_files/virt_large_slurm.yml index 08ec62472..bd741c6f9 100644 --- a/virtual/vars_files/virt_large_slurm.yml +++ b/virtual/vars_files/virt_large_slurm.yml @@ -16,7 +16,6 @@ slurm_build_dir_cleanup: false rsyslog_client_tcp_host: "{{ groups['slurm-metric'][0] }}" # Don't install these packages by default on large cluster -slurm_cluster_install_singularity: false slurm_cluster_install_cuda: false install_open_ondemand: false