Hello,

 

We have a two node GPU cluster with 8 NVidia GPUs. GRES is currently configured and works if a user defines it within their sbtach/interactive job submission (--gres=gpu:3). Users only have access to the GPUs they request. However, when they omit  “--gres=gpu:n”, they can use every GPU, which interferes with running jobs that used the gres option. I’m at a loss as to why this is happening. Can someone please look at our configuration to see if anything stands out?

SLURM Version = 21.08.5 

Slurm.conf

ClusterName=ommit

 

SlurmctldHost=headnode

 

ProctrackType=proctrack/cgroup

 

ReturnToService=2

 

SlurmdPidFile=/run/slurmd.pid

 

SlurmdSpoolDir=/var/lib/slurm/slurmd

 

StateSaveLocation=/var/lib/slurm/slurmctld

 

SlurmUser=slurm

 

TaskPlugin=task/cgroup

 

SchedulerType=sched/backfill

 

SelectType=select/cons_tres

 

SelectTypeParameters=CR_Core_Memory

 

AccountingStorageType=accounting_storage/slurmdbd

 

# AccountingStorageType for other resources

#

AccountingStorageTRES=gres/gpu

#DebugFlags=CPU_Bind,gres

 

JobCompType=jobcomp/none

 

JobAcctGatherType=jobacct_gather/cgroup

 

SlurmctldDebug=info

 

SlurmctldLogFile=/var/log/slurm/slurmctld.log

 

SlurmdDebug=info

 

SlurmdLogFile=/var/log/slurm/slurmd.log

 

DefMemPerCPU=4000

 

#NodeName=n01 CPUs=256 Boards=1 SocketsPerBoard=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=1000000

NodeName=n01 Gres=gpu:nvidia-l40:8 CPUs=256 Boards=1 SocketsPerBoard=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=1000000

NodeName=n02 Gres=gpu:nvidia-l40:8 CPUs=256 Boards=1 SocketsPerBoard=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=1000000

 

#Gres config for GPUs

GresTypes=gpu

 

PreemptType=preempt/qos

PreemptMode=REQUEUE

 

# reset usage after 1 week

PriorityUsageResetPeriod=WEEKLY

 

# The job's age factor reaches 1.0 after waiting in the

# queue for 2 weeks.

PriorityMaxAge=14-0

 

# This next group determines the weighting of each of the

# components of the Multifactor Job Priority Plugin.

# The default value for each of the following is 1.

PriorityWeightAge=1000

PriorityWeightFairshare=10000

PriorityWeightJobSize=1000

PriorityWeightPartition=1000

PriorityWeightQOS=1500

 

# Primary partitions

PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP

PartitionName=all Nodes=n01,n02 Default=YES MaxTime=01:00:00 DefaultTime=00:30:00 State=UP

PartitionName=statds Nodes=n01 Default=NO MaxTime=48:00:00 State=UP Priority=100 State=UP OverSubscribe=FORCE AllowAccounts=statds

PartitionName=phil Nodes=n02 Default=NO MaxTime=48:00:00 State=UP Priority=100 State=UP OverSubscribe=FORCE AllowAccounts=phil

 

#Set up condo mode

 

# Condo partitions

PartitionName=phil_condo Nodes=n02 Default=NO MaxTime=48:00:00 DefaultTime=00:01:00 State=UP Priority=50 OverSubscribe=FORCE AllowQos=normal

PartitionName=statds_condo Nodes=n01 Default=NO MaxTime=48:00:00 DefaultTime=00:01:00 State=UP Priority=50 OverSubscribe=FORCE AllowQos=normal

 

JobSubmitPlugins=lua

 

Gres.conf

NodeName=n01 Name=gpu Type=nvidia-l40 File=/dev/nvidia0

NodeName=n01 Name=gpu Type=nvidia-l40 File=/dev/nvidia1

NodeName=n01 Name=gpu Type=nvidia-l40 File=/dev/nvidia2

NodeName=n01 Name=gpu Type=nvidia-l40 File=/dev/nvidia3

NodeName=n01 Name=gpu Type=nvidia-l40 File=/dev/nvidia4

NodeName=n01 Name=gpu Type=nvidia-l40 File=/dev/nvidia5

NodeName=n01 Name=gpu Type=nvidia-l40 File=/dev/nvidia6

NodeName=n01 Name=gpu Type=nvidia-l40 File=/dev/nvidia7

NodeName=n02 Name=gpu Type=nvidia-l40 File=/dev/nvidia0

NodeName=n02 Name=gpu Type=nvidia-l40 File=/dev/nvidia1

NodeName=n02 Name=gpu Type=nvidia-l40 File=/dev/nvidia2

NodeName=n02 Name=gpu Type=nvidia-l40 File=/dev/nvidia3

NodeName=n02 Name=gpu Type=nvidia-l40 File=/dev/nvidia4

NodeName=n02 Name=gpu Type=nvidia-l40 File=/dev/nvidia5

NodeName=n02 Name=gpu Type=nvidia-l40 File=/dev/nvidia6

NodeName=n02 Name=gpu Type=nvidia-l40 File=/dev/nvidia7

 

 

Cgroup.conf

CgroupMountpoint="/sys/fs/cgroup"

CgroupAutomount=yes

CgroupReleaseAgentDir="/etc/slurm/cgroup"

AllowedDevicesFile="/etc/slurm/cgroup_allowed_devices_file.conf"

ConstrainCores=yes

ConstrainDevices=yes

ConstrainRAMSpace=yes

 

cgroup_allowed_devices_file.conf

/dev/null

/dev/urandom

/dev/zero

/dev/sda*

/dev/cpu/*/*

/dev/pts/*

/dev/nvidia*