SLURM
Controller Node 1. NTP 2. hostname
dnf install -y libaec dnf install -y slurm-slurmctld slurm-slurmdbd slurm-perlapi slurm-slurmrestd slurm-gui slurm-contribs
systemctl enable --now munge.service systemctl enable --now slurmdbd.service systemctl enable --now slurmctld.service
/etc/slurm/cgroup.conf
###
#
# Slurm cgroup support configuration file
#
# See man slurm.conf and man cgroup.conf for further
# information on cgroup configuration parameters
#--
CgroupAutomount=yes
ConstrainKmemSpace=no #avoid known Kernel issues
ConstrainCores=yes
ConstrainRAMSpace=yes
ConstrainSwapSpace=yes
AllowedRAMSpace=500
AllowedSwapSpace=0
MaxRAMPercent=100
MaxSwapPercent=25
TaskAffinity=no #use task/affinity plugin instead
/etc/slurm/slurm.conf
### slurm.conf controls the config for Slurm both in terms of the slurmctld
### daemon but also for slurmd on the clients. This also controls node and
### partition definition.
############################## Global #####################################
### This section sets global definitions
### Name of the cluster.
ClusterName=CLUSTERNAME
MailDomain=DOMAIN.TLD
### User that slurm will run as.
SlurmUser=slurm
### Sets munge to be our authentication method
AuthType=auth/munge
CryptoType=crypto/munge
### Width of the communications heirarchy. Recommended to be either N^(1/2)
### or N^(1/3) where N is the number of nodes.
### Currently we set TreeWidth to N^(1/3) due to the fact we are spanning
### multiple datacenters that have significant latency penalty.
#TreeWidth=12
#MessageTimeout=100
#RoutePlugin=route/topology
#TopologyPlugin=topology/tree
### We don't set a default version of MPI.
MpiDefault=none
SwitchType=switch/none
### scron settings
### Enable the use of scrontab to submit and manage periodic repeating jobs
ScronParameters=enable
########################### slurmctld #####################################
### This section sets the definitions specific to the slurmctld.
### Network info for slurmctld
SlurmctldHost=SLURMCTLD
SlurmctldPort=6817
SlurmctldPidFile=/run/slurm/slurmctld.pid
StateSaveLocation=/tools/slurm/var/spool/slurm/ctld
SlurmctldTimeout=120
SlurmctldParameters=preempt_send_user_signal,reboot_from_controller,user_resv_delete
RebootProgram="/usr/sbin/reboot"
### Logging
#SlurmctldDebug=verbose
SlurmctldDebug=debug
SlurmctldLogFile=/var/log/slurm/slurmctld.log
#SlurmctldLogFile=/var/log/messages
#SlurmctldSyslogDebug=verbose
#DebugFlags=Route
DebugFlags=Gres
### Prolog that the slurmctld runs when it schedules jobs.
#PrologSlurmctld=/usr/local/sbin/slurmctld_prolog
### Where are are logging data about job completions.
#JobCompHost=holy-slurm02
JobCompType=jobcomp/filetxt
JobCompLoc=/tools/slurm/var/JobComp.log
########################## slurmd #########################################
### This section sets the definitions specific to the slurmd and cgroups.
SlurmdPort=6818
#SrunPortRange=7845-11845
SlurmdPidFile=/run/slurm/slurmd.pid
SlurmdSpoolDir=/tools/slurm/var/spool/slurm/d
SlurmdTimeout=300
#SlurmdDebug=verbose
SlurmdDebug=debug
SlurmdLogFile=/var/log/slurm/slurmd.log
### Turning on config_overrides else the logs get filled up with reports of bad nodes.
#SlurmdParameters=config_overrides
### Logging
#SlurmdSyslogDebug=verbose
### We are using cgroups to track things and mange tasks.
ProctrackType=proctrack/cgroup
TaskPlugin=task/affinity,task/cgroup
### We have MCS turned on so that we can run isolated jobs.
#MCSPlugin=mcs/account
MCSPlugin=mcs/group
MCSParameters=ondemand,ondemandselect,privatedata
### What gres types we are using.
#GresTypes=gpu
GresTypes=mem_free
### Where scratch space is located on the node.
#TmpFs=/scratch
### Nodes will stay down until reopened by the admin.
### This prevents flapping by nodes.
#ReturnToService=0
ReturnToService=1
### This sets how the cgroup will be set up by the prolog script.
### We need X11 in order to use the Slurm PAM libraries and X11.
PrologFlags=Contain,X11
#UsePAM=1
### Location of Epilog Script
#Epilog=/usr/local/bin/slurm_epilog
### Node Health Check
#HealthCheckInterval=500
#HealthCheckNodeState=CYCLE
#HealthCheckProgram=/usr/local/bin/node_monitor
############################ slurmdbd #####################################
### This section sets the definitions for slurmdbd and accounting.
AccountingStorageHost=SLURMDBDHOST
AccountingStoragePort=6819
#AccountingStorageUser=slurm
AccountingStorageType=accounting_storage/slurmdbd
### Determines what we track for accounting.
### AccountingStorageEnforce being set to safe means that if we enforce a
### GrpCPUMins limit, jobs will only launch if they will complete before
### that limit is exhuasted. Else the jobs would simply terminate when the
### limit is hit regardless if the job still had time left.
AccountingStorageEnforce=safe
# DISABLING IB + LUSTRE ACCOUNTING FOR NOW UNTIL ISSUES RESOLVED
#AccountingStorageTRES=Billing,CPU,Energy,Mem,Node,FS/Disk,FS/Lustre,Pages,VMem,IC/OFED,gres/gpu
AccountingStorageTRES=Billing,CPU,Energy,Mem,Node,FS/Disk,Pages,VMem,gres/mem_free
AccountingStoreJobComment=YES
# DISABLING IB + LUSTRE ACCOUNTING FOR NOW UNTIL ISSUES RESOLVED
#AcctGatherInfinibandType=acct_gather_infiniband/ofed
#AcctGatherFilesystemType=acct_gather_filesystem/lustre
### Determines how frequently we will ping for job data.
### We use jobacct_gather/linux rather than cgroups as that is what is
### recommended by slurm in their documentation even though we have
### cgroups enabled.
JobAcctGatherType=jobacct_gather/linux
JobAcctGatherFrequency=30
#JobAcctGatherFrequency=task=30,network=30,filesystem=30
########################## Scheduling #####################################
### This section is specific to scheduling
### Tells the scheduler to enforce limits for all partitions
### that a job submits to.
EnforcePartLimits=ALL
### Let's slurm know that we have a jobsubmit.lua script
#JobSubmitPlugins=lua
### When a job is launched this locks the memory a user can use,
### it also sorts NUMA memory. In addition for srun invocations
### we test that the executable is actually available and has
### the correct permissions before launching.
LaunchParameters=mem_sort,slurmstepd_memlock_all,test_exec
### Set's licenses we are running.
### These to not tie into FlexLM or our License server.
#Licenses=lumerical:10,MATLAB_Distrib_Comp_Engine:256,renderman:29
Licenses=pcie_svt_free:195,low_priority:200
### Maximum sizes for Jobs.
#MaxJobCount=300000
#MaxArraySize=10000
#DefMemPerCPU=100
DefCpuPerGPU=1
#DefMemPerGPU=100
GpuFreqDef=low
DefMemPerNode=2600
### Job Timers
CompleteWait=0
### We set the EpilogMsgTime long so that Epilog Messages don't pile up all
### at one time due to forced exit which can cause problems for the master.
#EpilogMsgTime=3000000
InactiveLimit=0
KillWait=30
### This only applies to the reservation time limit, the job must still obey
### the partition time limit.
ResvOverRun=UNLIMITED
MinJobAge=300
Waittime=0
### Scheduling parameters
SchedulerType=sched/backfill
SelectType=select/cons_tres
SelectTypeParameters=CR_Core_Memory
### Govern's default preemption behavior
PreemptType=preempt/partition_prio
PreemptMode=REQUEUE
### default_queue_depth should be some multiple of the partition_job_depth,
### ideally number_of_partitions * partition_job_depth, but typically the main
### loop exits prematurely if you go over about 400. A partition_job_depth of
### 10 seems to work well.
SchedulerParameters=\
default_queue_depth=1500,\
partition_job_depth=10,\
bf_continue,\
bf_interval=30,\
bf_resolution=600,\
bf_window=11520,\
bf_max_job_part=0,\
bf_max_job_user=10,\
bf_max_job_test=100000,\
bf_max_job_start=1000,\
bf_ignore_newly_avail_nodes,\
enable_user_top,\
pack_serial_at_end,\
nohold_on_prolog_fail,\
permit_job_expansion,\
preempt_strict_order,\
preempt_youngest_first,\
reduce_completing_frag,\
requeue_setup_env_fail,\
max_rpc_cnt=16
### This controls how we handle dependencies. If we have invalid dependencies
### we terminate the job.
DependencyParameters=kill_invalid_depend
################################ Fairshare ################################
### This section sets the fairshare calculations
PriorityType=priority/multifactor
PriorityFlags=NO_FAIR_TREE
### Settings for fairshare calculation frequency and shape.
FairShareDampeningFactor=1
PriorityDecayHalfLife=28-0
PriorityCalcPeriod=5
### Settings for fairshare weighting.
PriorityMaxAge=7-0
PriorityWeightAge=1000
PriorityWeightFairshare=2000
PriorityWeightJobSize=0
PriorityWeightPartition=0
PriorityWeightQOS=10000
###########################################################################
############################## Node Definitions ###########################
### These are ordered in alphabetical order by node name.
### The AMD nodes do not match the actual hardware specs
### but should get us more job slots.
### Header names are names of those who bought the hardware.
###########################################################################
##########################Partition Definitions ###########################
### Organized in alphabetical order by group name.
### Then partitions are ordered by partition name.
### Nodes in the partition and groups should be ordered alphabetically.
### TRES Billing formula for CPU: AMD = 0.1, Sandy = 0.2, Broadwell = 0.4
### Skylake = 0.5, Cascade Lake = 1.0
### TRES Billing formula for GPU: TitanX = 2.2, K80 = 15.4, V100 = 75
### TRES Billing formula for Memory: NumCPU*CPUTRES/TotalMem
PartitionName=test State=UP PreemptMode=OFF PriorityTier=1 \
Nodes=all
### Arguelles Delgado, Physics, FAS ###
### holy7c183[01-11] is on loan from shared ###
###PartitionName=arguelles_delgado State=UP PreemptMode=OFF PriorityTier=4 DefaultTime=0-00:10:00 \
### TRESBillingWeights="CPU=1.0,Mem=0.25G" \
### AllowGroups=arguelles_delgado_lab,slurm-admin \
### Nodes=holy7c183[01-11]
/etc/slurm/slurmdbd.conf
#
# See the slurmdbd.conf man page for more information.
#
# Archive info
#ArchiveJobs=yes
#ArchiveDir="/tmp"
#ArchiveSteps=yes
#ArchiveScript=
#JobPurge=12
#StepPurge=1
#
# Authentication info
AuthType=auth/munge
#AuthInfo=/var/run/munge/munge.socket.2
#
# slurmdbd info
#DebugLevel=4
DebugLevel=verbose
#DefaultQOS=normal,standby
DbdAddr=localhost
DbdHost=SLURMDBDHOST
#DbdPort=6819
DbdPort=6819
#LogFile=
LogFile=/var/log/slurm/slurmdbd.log
#MessageTimeout=300
PidFile=/var/run/slurm/slurmdbd.pid
#PluginDir=
#PrivateData=accounts,users,usage,jobs
PurgeEventAfter=999month
PurgeJobAfter=999month
PurgeResvAfter=999month
PurgeStepAfter=999month
PurgeSuspendAfter=999month
PurgeTXNAfter=999month
PurgeUsageAfter=999month
#SlurmUser=root
SlurmUser=slurm
#TrackWCKey=yes
#
# Database info
StorageType=accounting_storage/mysql
StorageHost=localhost
StoragePort=3306
StoragePass=DBPASSWORD
StorageUser=slurm
StorageLoc=slurm_acct_db
#StorageLoc=slurm_acct_db