<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:#954F72;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
p.msonormal0, li.msonormal0, div.msonormal0
{mso-style-name:msonormal;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:0in;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
span.EmailStyle19
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle20
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:#1F497D;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:1639070166;
mso-list-template-ids:-140246568;}
@list l0:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1
{mso-list-id:1676956461;
mso-list-type:hybrid;
mso-list-template-ids:-1783171012 -384004384 201916419 201916421 201916417 201916419 201916421 201916417 201916419 201916421;}
@list l1:level1
{mso-level-start-at:0;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:20.25pt;
text-indent:-.25in;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:56.25pt;
text-indent:-.25in;
font-family:"Courier New";}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:92.25pt;
text-indent:-.25in;
font-family:Wingdings;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:128.25pt;
text-indent:-.25in;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:164.25pt;
text-indent:-.25in;
font-family:"Courier New";}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:200.25pt;
text-indent:-.25in;
font-family:Wingdings;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:236.25pt;
text-indent:-.25in;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:272.25pt;
text-indent:-.25in;
font-family:"Courier New";}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
margin-left:308.25pt;
text-indent:-.25in;
font-family:Wingdings;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-US" link="#0563C1" vlink="#954F72">
<div class="WordSection1">
<p class="MsoNormal"><span style="color:#1F497D">The times for the two runs suggest that the version run through slurm is using only one core.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D"><o:p> </o:p></span></p>
<div>
<p class="MsoNormal"><span style="color:#1F497D">Best – Don<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">Don Krieger, PhD<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">Research Scientist<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">Department of Neurological Surgery<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">University of Pittsburgh</span><span style="color:#1F497D"><o:p></o:p></span></p>
</div>
<p class="MsoNormal"><span style="color:#1F497D"><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal"><b>From:</b> slurm-users <slurm-users-bounces@lists.schedmd.com>
<b>On Behalf Of </b>Williams, Gareth (IM&T, Black Mountain)<br>
<b>Sent:</b> Tuesday, December 15, 2020 4:51 PM<br>
<b>To:</b> Slurm User Community List <slurm-users@lists.schedmd.com><br>
<b>Subject:</b> Re: [slurm-users] Scripts run slower in slurm?<o:p></o:p></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal"><span lang="EN-AU">Hi John,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU">I’ll volunteer an opinion. There are circumstances where slurm could contribute to slower overall times for tasks, as slurm can be configured to do pre-job setup and post-job followup (prologue/epilogue). However, you
are reporting within-task timing, not overall timing so this is beside the point.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU">In general if running ‘the same code’ gives different results (or different performance) then the difference will be in the environment/context. Slurm is a part of this context but probably not directly related… More
likely perhaps are:<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:20.25pt;text-indent:-.25in;mso-list:l1 level1 lfo3">
<![if !supportLists]><span lang="EN-AU"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-AU">Hardware variations<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:20.25pt;text-indent:-.25in;mso-list:l1 level1 lfo3">
<![if !supportLists]><span lang="EN-AU"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-AU">Contention for cpu/memory/network<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:20.25pt;text-indent:-.25in;mso-list:l1 level1 lfo3">
<![if !supportLists]><span lang="EN-AU"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-AU">cgroup constraints (And other OS differences)<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:20.25pt;text-indent:-.25in;mso-list:l1 level1 lfo3">
<![if !supportLists]><span lang="EN-AU"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-AU">IO connectivity differences (and caching)<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:20.25pt;text-indent:-.25in;mso-list:l1 level1 lfo3">
<![if !supportLists]><span lang="EN-AU"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-AU">Maybe effects from past activity – is extra work needed to free memory or cache content – special case of contention.<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:20.25pt;text-indent:-.25in;mso-list:l1 level1 lfo3">
<![if !supportLists]><span lang="EN-AU"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-AU">Code differences (possibly from dynamically loaded libraries or a different interpreter/python version).<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:20.25pt;text-indent:-.25in;mso-list:l1 level1 lfo3">
<![if !supportLists]><span lang="EN-AU"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-AU">I guess shell environment settings could matter (probably not in your case and probably not one you are within python).<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU">In your case, perhaps system calls are slower in one context vs another. t.time() might be slower.
<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU">You might use strace -tttT … to see if there are slow system calls in one context vs the other.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU">Good luck!<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU">Gareth<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
<p class="MsoNormal"><b>From:</b> slurm-users <<a href="mailto:slurm-users-bounces@lists.schedmd.com">slurm-users-bounces@lists.schedmd.com</a>>
<b>On Behalf Of </b>Alpha Experiment<br>
<b>Sent:</b> Tuesday, 15 December 2020 6:20 PM<br>
<b>To:</b> Slurm User Community List <<a href="mailto:slurm-users@lists.schedmd.com">slurm-users@lists.schedmd.com</a>><br>
<b>Subject:</b> [slurm-users] Scripts run slower in slurm?<o:p></o:p></p>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
<div>
<p class="MsoNormal"><span lang="EN-AU">Hi,<br>
<br>
I made a short script in python to test if slurm was correctly limiting the number of CPUs available to each partition. The script is as follows:<br>
</span><span lang="EN-AU" style="font-family:"Courier New"">import multiprocessing as mp<br>
import time as t<br>
<br>
def fibonacci(n):<br>
n = int(n)<br>
def fibon(a,b,n,result):<br>
c = a+b<br>
result.append(c)<br>
if c < n:<br>
fibon(b,c,n,result)<br>
return result<br>
return fibon(0,1,n,[])<br>
<br>
def calcnfib(n):<br>
res = fibonacci(n)<br>
return res[-1]</span><span lang="EN-AU"><o:p></o:p></span></p>
<div>
<p class="MsoNormal"><span lang="EN-AU" style="font-family:"Courier New""><br>
def benchmark(pool):<br>
t0 = t.time()<br>
out = pool.map(calcnfib, range(1000000, 1000000000,1000))<br>
tf = t.time()<br>
return str(tf-t0)<br>
<br>
pool = mp.Pool(4)<br>
print("4: " + benchmark(pool))<br>
<br>
pool = mp.Pool(32)<br>
print("32: " + benchmark(pool))<br>
<br>
pool = mp.Pool(64)<br>
print("64: " + benchmark(pool))<br>
<br>
pool = mp.Pool(128)<br>
print("128: " + benchmark(pool))</span><span lang="EN-AU"><br>
<br>
It is called using the following submission script:<br>
</span><span lang="EN-AU" style="font-family:"Courier New"">#!/bin/bash<br>
#SBATCH --partition=full<br>
#SBATCH --job-name="Large"<br>
source testenv1/bin/activate<br>
python3 <a href="https://urldefense.com/v3/__http:/multithread_example.py__;!!NHLzug!f2hZoOMCdFDo0hYtXiwmPvbUgKnGlY4b91eWH9pNDbIfGKH7KyDFoAfIBWcAk3Pd$">
multithread_example.py</a></span><span lang="EN-AU"><br>
<br>
The slurm out file reads<br>
</span><span lang="EN-AU" style="font-family:"Courier New"">4: 5.660163640975952<br>
32: 5.762076139450073<br>
64: 5.8220226764678955<br>
128: 5.85421347618103</span><span lang="EN-AU"><br>
<br>
However, if I run<br>
</span><span lang="EN-AU" style="font-family:"Courier New"">source testenv1/bin/activate<br>
python3 <a href="https://urldefense.com/v3/__http:/multithread_example.py__;!!NHLzug!f2hZoOMCdFDo0hYtXiwmPvbUgKnGlY4b91eWH9pNDbIfGKH7KyDFoAfIBWcAk3Pd$">
multithread_example.py</a></span><span lang="EN-AU"><o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-AU"><br>
I find faster and more expected behavior<br>
</span><span lang="EN-AU" style="font-family:"Courier New"">4: 1.5878620147705078<br>
32: 0.34162330627441406<br>
64: 0.24987316131591797<br>
128: 0.2247719764709472</span><span lang="EN-AU"><br>
<br>
For reference my slurm configuration file is<br>
</span><span lang="EN-AU" style="font-family:"Courier New""># slurm.conf file generated by configurator easy.html.<br>
# Put this file on all nodes of your cluster.<br>
# See the slurm.conf man page for more information.<br>
#<br>
#SlurmctldHost=localhost<br>
ControlMachine=localhost<br>
<br>
#MailProg=/bin/mail<br>
MpiDefault=none<br>
#MpiParams=ports=#-#<br>
ProctrackType=proctrack/cgroup<br>
ReturnToService=1<br>
SlurmctldPidFile=/home/slurm/run/<a href="https://urldefense.com/v3/__http:/slurmctld.pid__;!!NHLzug!f2hZoOMCdFDo0hYtXiwmPvbUgKnGlY4b91eWH9pNDbIfGKH7KyDFoAfIBT0j-fRd$">slurmctld.pid</a><br>
#SlurmctldPort=6817<br>
SlurmdPidFile=/home/slurm/run/<a href="https://urldefense.com/v3/__http:/slurmd.pid__;!!NHLzug!f2hZoOMCdFDo0hYtXiwmPvbUgKnGlY4b91eWH9pNDbIfGKH7KyDFoAfIBcacS8sC$">slurmd.pid</a><br>
#SlurmdPort=6818<br>
SlurmdSpoolDir=/var/spool/slurm/slurmd/<br>
SlurmUser=slurm<br>
#SlurmdUser=root<br>
StateSaveLocation=/home/slurm/spool/<br>
SwitchType=switch/none<br>
TaskPlugin=task/affinity<br>
<br>
# TIMERS<br>
#KillWait=30<br>
#MinJobAge=300<br>
#SlurmctldTimeout=120<br>
#SlurmdTimeout=300<br>
<br>
# SCHEDULING<br>
SchedulerType=sched/backfill<br>
SelectType=select/cons_tres<br>
SelectTypeParameters=CR_Core<br>
<br>
# LOGGING AND ACCOUNTING<br>
AccountingStorageType=accounting_storage/none<br>
ClusterName=cluster<br>
#JobAcctGatherFrequency=30<br>
JobAcctGatherType=jobacct_gather/none<br>
#SlurmctldDebug=info<br>
SlurmctldLogFile=/home/slurm/log/slurmctld.log<br>
#SlurmdDebug=info<br>
#SlurmdLogFile=</span><span lang="EN-AU"><o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-AU" style="font-family:"Courier New""><br>
# COMPUTE NODES<br>
NodeName=localhost CPUs=128 RealMemory=257682 Sockets=1 CoresPerSocket=64 ThreadsPerCore=2 State=UNKNOWN<br>
PartitionName=full Nodes=localhost Default=YES MaxTime=INFINITE State=UP<br>
PartitionName=half Nodes=localhost Default=NO MaxTime=INFINITE State=UP MaxNodes=1 MaxCPUsPerNode=64 MaxMemPerNode=128841</span><span lang="EN-AU"><o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
</div>
<p class="MsoNormal"><span lang="EN-AU" style="font-family:"Arial",sans-serif">Here is my cgroup.conf file as well</span><span lang="EN-AU"><br>
</span><span lang="EN-AU" style="font-family:"Courier New"">CgroupAutomount=yes</span><span lang="EN-AU"><br>
</span><span lang="EN-AU" style="font-family:"Courier New"">ConstrainCores=no</span><span lang="EN-AU"><br>
</span><span lang="EN-AU" style="font-family:"Courier New"">ConstrainRAMSpace=no</span><span lang="EN-AU"><o:p></o:p></span></p>
<div>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-AU">If anyone has any suggestions for what might be going wrong and why the script takes much longer when run with slurm, please let me know!<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-AU"><o:p> </o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-AU">Best,<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-AU">John<o:p></o:p></span></p>
</div>
</div>
</div>
</body>
</html>