
    #ShSK              	          d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
Z
d dlZddlmZ ddlmZmZmZmZ ddlmZ dd	lmZ dPdeej                 dededdfdZe j        dej        ej                 fd            ZdeddfdZdQdZdeddfdZe
j         !                    dd          de"deddfd            Z#deddfdZ$e j        dRdej%        e&         de"fd            Z'dedej(        fdZ)deddfdZ*deddfdZ+deddfd Z,dQd!Z-dQd"Z.dQd#Z/deddfd$Z0deddfd%Z1dQd&Z2e
j         !                    d'g d(          d)ed*eddfd+            Z3e
j         !                    d,d-d.gfd/d0gfd1d2gfd3g d4fd5d6gfg          d7ed8ej4        ej5        ej6        e&ef         d9f                  ddfd:            Z7e
j         !                    d;d<d=g          d>ed8eddfd?            Z8dQd@Z9dQdAZ:dQdBZ;dQdCZ<e j        dDedej        ej=                 fdE            Z>dQdFZ?dQdGZ@dQdHZAdQdIZBdJeddfdKZCe
j         !                    dLi dMdig          dLejD        ee&f         deddfdN            ZEdeddfdOZFdS )S    N)Path)patch   )helpers)job_environment
submission	test_coreutils)Job   )slurm jobprintserrorsreturnc                      fd j         j         j         j        fD             }t          |||f          D ]T\  }}t	          |                              d          5 }|                    |           ddd           n# 1 swxY w Y   UdS )zWrite fake log filesc                 z    g | ]7}t          |                              d t          j                            8S )%j)strreplacejob_id).0xr   s     _/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/submitit/slurm/test_slurm.py
<listcomp>z#_mock_log_files.<locals>.<listcomp>   s5    eee1Qc#*oo66eee    wN)pathsstdoutstderrzipr   openwrite)r   r   r   	filepathsfilepathmsgfs   `      r   _mock_log_filesr)      s    eeeeAQSVS\Sc@deeeIY(899  #(^^  %% 	GGCLLL	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 s   BB	B	c               #   .  K   t          j        dg          } 	 |                                 5  | V  d d d            n# 1 swxY w Y   t          j        j                                         d S # t          j        j                                         w xY w)Nsrun)
known_cmds)r	   MockedSubprocesscontextr   SlurmJobwatcherclearmocks    r   mocked_slurmr4       s      %&:::D'\\^^ 	 	JJJ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	$$&&&&&$$&&&&s+   A/ ?A/ AA/ AA/ /%Btmp_pathc                     t                      5 }|                    dd           t          j        | d          }|j        dk    sJ |                    d           d d d            d S # 1 swxY w Y   d S )N12z       UNKNOWNFtimeout)r4   set_job_stater   r/   state
_interrupt)r5   r3   r   s      r   test_mocked_missing_stater>   +   s    	 &44+++$)N8T$B$ByI%%%%u%%%	& & & & & & & & & & & & & & & & & &s   AA++A/2A/c                     t                      5 } |                     dd           |                     d          5  t          j                    j        dk    sJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr7   RUNNINGr   )r4   r;   job_contextr   JobEnvironmentclusterr2   s    r   test_job_environmentrD   3   s0   	 G44+++d## 	G 	G"133;wFFFFF	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	GG G G G G G G G G G G G G G G G G Gs4   ,A?A'A?'A+	+A?.A+	/A??BBc                 |   t                      5 }t          j        |           }|                    t          j        ddd          }|j        dk    sJ |j        dk    sJ |                                J t          |dd	
           d|_
        t          j        t          j                  5  |                                 d d d            n# 1 swxY w Y   t          |dd	
           |                    |j                  5  t#          j        |j        j                   d d d            n# 1 swxY w Y   |                                dk    sJ |                                d	k    sJ |                                dk    sJ dt.          j        vs
J d            	 d d d            d S # 1 swxY w Y   d S )Nfolderr   r      blublur7   r@   This is the error log
hop)r   r   r      _USELESS_TEST_ENV_VAR_z(Test context manager seems to be failing)r4   r   SlurmExecutorsubmitr	   
do_nothingr   r<   r    r)   _results_timeout_spytestraisesr
   UncompletedJobError_get_outcome_and_resultrA   r   process_jobr   rG   resultr!   osenviron)r5   r3   executorr   s       r   test_slurm_job_mockedr\   :   s   	 f4&h777ooi2AqoCCzT!!!!yI%%%%zz||###$=eLLLL!"]5455 	* 	*'')))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*$=eLLLLcj)) 	5 	5"39#3444	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5zz||r!!!!zz||u$$$$zz||88888'rz999;e9999'f f f f f f f f f f f f f f f f f fs[   B F1/CF1C	F1C	/F1D2&F12D6	6F19D6	:A)F11F58F5use_batch_api)FTc           
         d}t                      5 }t          j        |          }|                    d           t	          |          t	          dd|z             cdt
          dt
          dt
          ffd	}g }| rn|                                5  t                    D ]/\  }}|                    |	                    |||                     0	 d d d            n# 1 swxY w Y   n|
                    |          }|d
         j                            d          d
         fdt	          |          D             d |D             k    sJ |D ]_}	|	j        dk    sJ |                    |	j                  5  t          j        |	j        j                   d d d            n# 1 swxY w Y   `|d
                             d           t)          t+          |                    d |D             k    sJ t-          |          j        j                                        }
d |
                                D             }|dgk    sJ 	 d d d            d S # 1 swxY w Y   d S )N   rF   rH   )array_parallelism
   r   yr   c                 &    | v sJ |v sJ | |z   S N )r   rb   data1data2s     r   addz(test_slurm_job_array_mocked.<locals>.addY   s&    ::::::::q5Lr   r   _c                     g | ]	} d | 
S )ri   re   )r   aarray_ids     r   r   z/test_slurm_job_array_mocked.<locals>.<listcomp>f   s'    444a8!!a!!444r   c                     g | ]	}|j         
S re   r   r   js     r   r   z/test_slurm_job_array_mocked.<locals>.<listcomp>f   s    8P8P8Pa8P8P8Pr   r@   Tr9   c                 6    g | ]}|                                 S re   )rX   ro   s     r   r   z/test_slurm_job_array_mocked.<locals>.<listcomp>n   s     /I/I/Iq

/I/I/Ir   rn   c                 >    g | ]}d |v |                                 S )z--array)strip)r   ls     r   r   z/test_slurm_job_array_mocked.<locals>.<listcomp>q   s%    OOOA	Qaggiir   z#SBATCH --array=0-4%3)r4   r   rO   update_parametersrangeintbatchr"   appendrP   	map_arrayr   splitr<   rA   r   rW   r   rG   r=   listmapr   submission_file	read_text
splitlines)r]   r5   nr3   r[   rh   jobsd1d2r   sbatch
array_linerl   rf   rg   s               @@@r   test_slurm_job_array_mockedr   Q   ss   	A	 74&h777""Q"777Qxxr26!2!2u	3 	3 	3 	 	 	 	 	 	 	
 #% 	9!! > >!%// > >FBKKR < <====>> > > > > > > > > > > > > > > %%c5%88D7>'',,Q/44445884448P8P48P8P8PPPPP 	9 	9C9	))))!!#*-- 9 9&sy'78889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 	Q4(((CUE**++/I/ID/I/I/IIIIIXh///5EOOQQOO):):)<)<OOO
5666666=7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7s]   BI'AC%I'%C)	)I',C)	-BI'F/#I'/F33I'6F37B"I''I+.I+c                    t                      5 }t          j        |           }|                    dd           |                    t
          j        ddd          }|                    |j                  5  t          j
        t                    5  t          j        |j        j                   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t!          |d	
           t          j
        t"          j                  5  |                                 d d d            n# 1 swxY w Y   |                                }t+          |t"          j                  sJ 	 d d d            d S # 1 swxY w Y   d S )NrF      r   )timegpus_per_noder   r   rM   )errorrK   )r   )r4   r   rO   ru   rP   r	   rQ   rA   r   rS   rT   
ValueErrorr   rW   r   rG   r)   r
   FailedJobErrorrX   	exception
isinstance)r5   r3   r[   r   r   s        r   test_slurm_error_mockedr   u   sO   	 ;4&h777""!"<<<ooi2AqoCCcj)) 	9 	9z** 9 9&sy'78889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	$=>>>>]5/00 	 	JJLLL	 	 	 	 	 	 	 	 	 	 	 	 	 	 	MMOO	)U%9::::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;s~   A*E99CB>2C>CCCC	E9C	E9C	2E9D0$E90D4	4E97D4	83E99E= E=Fcalled_with
not_calledc              #   4  K   |s| J t          dd           }|5 }	 d V  |r|                                 nG|                    |            n1# |r|                                 w |                    |            w xY wd d d            d S # 1 swxY w Y   d S )Nz1submitit.slurm.slurm.SlurmJobEnvironment._requeue)return_value)r   assert_not_calledassert_called_with)r   r   requeue_patchs       r   mock_requeuer      s     0000GVZ[[[G	 7F	7EEE 7((****))+6666  7((****))+66667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7s'   BA-B.A==BBBc                     t          j                    }t          j                            | j        j                  }t          j        || j        |          }|S rd   )	r   SlurmJobEnvironmentr
   DelayedSubmissionloadr   submitted_pickler   SignalHandler)r   envdelayedsigs       r   get_signal_handlerr      sF    

#
%
%C%**39+EFFG

'SY
@
@CJr   c                 0   t           j                                        }t          j                    }|                    t          j        d           t          |t          j	                  sJ t                      5  t          j        | d          }|                    d           |                    |          }d d d            n# 1 swxY w Y   t          |          } |d           t!          j        t$                    5  t'          d          5  |                    |           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          |          } |d	           t!          j        t$                    5  t'          d
          5  |                    |           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          |          } |d           t          j                                        }t'          d          5  t!          j        t,          j        d          5  |                    |           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nra   r   rG   max_num_timeout<   r      minutesr   2   r   7   Tr   ztimed-out too many times.match)submititrB   _usr_sigr   FunctionSequencerh   r	   _three_timer   Checkpointabler4   r   rO   ru   rP   r   rS   rT   
SystemExitr   checkpoint_and_try_requeuer   r
   rU   )r5   fast_forward_clockusr_sigfs0r[   r   r   s          r   test_requeuing_checkpointabler      s.   %..00G

"
$
$CGGI!2&&&c7122222 
 # #&hJJJ"""+++ooc""# # # # # # # # # # # # # # #
 S
!
!Cr""""	z	"	" 0 0LQ$?$?$? 0 0&&w///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 S
!
!Cr"""" 
z	"	" 0 0LQ$?$?$? 0 0&&w///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 S
!
!Cr"""" '0022G		&	&	& 0 0!)D) ) ) 0 0 	&&w///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   <AC

CC	ED<0E<E 	 EE 	EEEG#G9GG			GG		GG #G 0!JI3'J3I7	7J:I7	;JJJc                    t           j                                        }t                      5  t	          j        | d          }|                    d           |                    t          j	        d          }d d d            n# 1 swxY w Y   t          |          } |d           t          d	          5  |                    t          j        j                   d d d            n# 1 swxY w Y   t!          j        t$                    5  t          d
          5  |                    |           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          |          } |d           t          d	          5  t!          j        t(          j        d          5  |                    |           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   ra   r   r   Tr   r   r   z timed-out and not checkpointabler   )r   rB   r   r4   r   rO   ru   rP   r	   r   r   r   bypasssignalSignalsSIGTERMrS   rT   r   r   r
   rU   )r5   r   r   r[   r   r   s         r   !test_requeuing_not_checkpointabler      s[   %..00G	 9 9&hJJJ"""+++ooi3R889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 S
!
!Cr""""		&	&	& + +

6>)***+ + + + + + + + + + + + + + + 
z	"	" 0 0LQ$?$?$? 0 0&&w///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 S
!
!Cr"""" 
	&	&	& 0 0!)K) ) ) 0 0 	&&w///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   ABB
B
<%C--C14C1E"E8EE	EE	EE"E!G,2GG,G	G,G	G,,G03G0c                    t           j                                        }t                      5  t	          j        | d          }|                    d           |                    t          j	        d          }d d d            n# 1 swxY w Y   t          |          }t          j        t                    5  t          d          5  |                    |           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t           j                            |j        j                  }|j        dk    sJ d S )Nr   r   r   r   ra   Tr   )r   rB   r   r4   r   rO   ru   rP   r	   r   r   rS   rT   r   r   checkpoint_and_exitr
   r   r   r   r   _timeout_countdown)r5   r   r[   r   r   r   s         r   test_checkpoint_and_exitr      s   %..00G	 9 9&hJJJ"""+++ooi3R889 9 9 9 9 9 9 9 9 9 9 9 9 9 9
 S
!
!C	z	"	" ) )LD$A$A$A ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) %**39+EFFG%******sH   ABB
B
9D
C, D,C0	0D3C0	4DD
Dc            
         t          j        ddddddig d          } d	| v sJ d
| vsJ d| vsJ t          t                    j        dz  }|                                s|                    |            |                                }g }t          t          | 
                                |
                                                    D ]0\  }\  }}||k    r"|                    d|dz    d| d| d           1|r`t          |            t          d           t          |           dg|z   }|ddd| gz  }t          d                    |                    d S )Nz
blublu bar/tmp	learnfairTrJ   rM   )z-vvz
--cpu-bindnone)commandrG   	partition	exclusiveadditional_parameters	srun_argsr   z	--command
constraintz_sbatch_test_record.txtzline #r   z: "z" -> ""z	# # # # #zDifference with reference file:r   z)Delete the record file if this is normal:zrm 
)r   _make_sbatch_stringr   __file__parentexists
write_textr   	enumerater"   r   ry   printAssertionErrorjoin)stringrecord_filerecordedchangeskline1line2messages           r   test_make_sbatch_stringr      s   &'n///  F &    f$$$$v%%%%x..'*CCK 'v&&&$$&&HG&s6+<+<+>+>@S@S@U@U'V'VWW E E>E5E>>NNCAECCeCC5CCCDDD 1fkh45?BCEX;EXEXYYTYYw//0001 1r   c                  @    t          j        ddd          } d| v sJ d S )NrJ   r   r   )r   rG   r   z--gpus-per-node=2r   r   r   s    r   test_make_sbatch_string_gpur     s1    &xVWXXXF&((((((r   c                  @    t          j        ddd          } d| vsJ d S )NrJ   r   T)r   rG   stderr_to_stdout--errorr   r   s    r   test_make_sbatch_stderrr     s1    &xY]^^^FF""""""r   c                     t                      5  t          j        |           }d d d            n# 1 swxY w Y   |                    d           |j        j        d         dk    sJ d S )NrF   g      @)mem_gbmem3584MB)r4   r   AutoExecutorru   	_executor
parametersr5   r[   s     r   test_update_parametersr     s    	 : :(999: : : : : : : : : : : : : : :c***(/8;;;;;;s   155c                    t                      5  t          j        |           }d d d            n# 1 swxY w Y   t          j        t
                    5  |                    d           d d d            d S # 1 swxY w Y   d S )NrF   rM   rI   )r4   r   rO   rS   rT   r   ru   r   s     r   test_update_parameters_errorr     s    	 8 8&h7778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	z	"	" . .""""---. . . . . . . . . . . . . . . . . .s   155A99A= A=c                      d} t          j                                        |           }|d         dddk    sJ |d         dddk    sJ t          |          h dk    sJ d S )	NzoJobID|State
5610980|RUNNING
5610980.ext+|RUNNING
5610980.0|RUNING
20956421_0|RUNNING
20956421_[2-4%25]|PENDING
5610980r@   )JobIDState
20956421_2z20956421_[2-4%25]PENDING>   r   
20956421_0r   
20956421_3
20956421_4)r   SlurmInfoWatcher	read_infoset)exampleoutputs     r   test_read_infor  &  s    G #%%//88F))i H HHHHH,-@9#U#UUUUUv;;]]]]]]]]]r   z
name,state))12_0R)12_1U)12_2X)12_3r  )12_4r  namer<   c                     d}t          j                    }dD ]}|                    |           |                    |          }|                    | i                               dd          |k    sJ d S )Nz JobID|State
12_0|R
12_[2,4-12]|X)r  r  r   r  )r   r  register_jobr  get)r  r<   r  r0   jobidr  s         r   test_read_info_arrayr  4  s     3G$&&G! $ $U####w''F::dB##GS11U::::::r   zjob_id,expectedr7   )rM   r  )rM   r   z20_[2-7%56]   r      z20_[2-7,12-17,22%56])r  )r  rM      )r     z20_[0%1])r  r   r   expected.c                 R    t          j        |           }|d |D             k    sJ d S )Nc                 @    g | ]}t          d  |D                       S )c              3   4   K   | ]}t          |          V  d S rd   )r   r   r   s     r   	<genexpr>z.test_read_job_id.<locals>.<listcomp>.<genexpr>L  s(      22qCFF222222r   )tuple)r   groups     r   r   z$test_read_job_id.<locals>.<listcomp>L  s/    IIIue22E22222IIIr   )r   read_job_id)r   r  r  s      r   test_read_job_idr$  @  s:     v&&FIIIIIIIIIIIr   zstring,expected)s   Submitted batch job 5610208
5610208)zSubmitted batch job 5610208
r%  r   c                 T    t           j                            |           }||k    sJ d S rd   )r   rO   #_get_job_id_from_submission_command)r   r  r  s      r   #test_get_id_from_submission_commandr(  O  s1    
  DDVLLFXr   c                      t          j        t          j                  5  t          j                            d           d d d            d S # 1 swxY w Y   d S )Ns   blublur   )rS   rT   r
   FailedSubmissionErrorr   rO   r'  re   r   r   )test_get_id_from_submission_command_raiser+  X  s    	u2	3	3 R R??y?QQQR R R R R R R R R R R R R R R R R Rs   !AAAc                     t                      5 } t          j                    }|                     dd           |j        dk    sJ |                    d          }t          |j                                                  dhk    sJ |j	        dhk    sJ |dk    sJ |                     dd           |                    dd	          }|dk    sJ |j	        ddhk    sJ |j
        dhk    sJ 	 d d d            d S # 1 swxY w Y   d S )
Nr7   r@   r   11rn   r8   FAILEDforce)r   mode)r4   r   r  r;   	num_calls	get_stater  
_info_dictkeys_registered	_finished)r3   r0   r<   s      r   test_watcherr7  ]  s|   	 +4(**4+++ A%%%%!!!..7%**,,--$7777"tf,,,,	!!!!4***!!G!<<    "tTl2222 TF*****+ + + + + + + + + + + + + + + + + +s   C"C??DDc                  H    t          j                    } | d         dk    sJ d S )Nnodesr   )r   _get_default_parameters)defaultss    r   test_get_default_parametersr<  o  s,    ,..HG!!!!!!r   c                  N    t           j                                        dk    sJ d S )Nr   )r   rO   r  re   r   r   	test_namer>  t  s(    ##%%000000r   	node_listc              #      K   dt           j        d<   | t           j        d<   t          j                    V  t           j        d= t           j        d= d S )N1SLURM_JOB_IDSLURM_JOB_NODELIST)rY   rZ   r   r   )r?  s    r   with_slurm_job_nodelistrD  x  sO      !$BJ~'0BJ#$

#
%
%%%%

'(

>"""r   c                  R   t          d          5 } dg| j        k    sJ 	 d d d            n# 1 swxY w Y   t          d          5 } ddg| j        k    sJ 	 d d d            n# 1 swxY w Y   t          d          5 } ddg| j        k    sJ 	 d d d            n# 1 swxY w Y   t          d          5 } d	d
g| j        k    sJ 	 d d d            n# 1 swxY w Y   t          d          5 } g d| j        k    sJ 	 d d d            n# 1 swxY w Y   t          d          5 } g d| j        k    sJ 	 d d d            n# 1 swxY w Y   t          d          5 } g d| j        k    sJ 	 d d d            n# 1 swxY w Y   t          d          5 } g d| j        k    sJ 	 d d d            n# 1 swxY w Y   t          d          5 } ddg| j        k    sJ 	 d d d            n# 1 swxY w Y   t          d          5 } g d| j        k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nzcompute-b24zcompute-a1,compute-b2
compute-a1z
compute-b2zcompute-b2[1,2]compute-b21compute-b22zcompute-b2[011,022]zcompute-b2011zcompute-b2022zcompute-b2[1-3])rG  rH  compute-b23zcompute-b2[1-3,5,6,8])rG  rH  rI  compute-b25compute-b26compute-b28zcompute-b2[1-3,5-6,8]z compute-b2[1-3,5-6,8],compute-a1)rG  rH  rI  rJ  rK  rL  rF  zcompute[042,044]
compute042
compute044zcompute[042-043,045,048-049])rM  
compute043
compute045
compute048
compute049rD  	hostnamesr   s    r   test_slurm_node_listrV    s^   	 	/	/ 03#-/////0 0 0 0 0 0 0 0 0 0 0 0 0 0 0	 !8	9	9 =Sl+s}<<<<<= = = = = = = = = = = = = = =	 !2	3	3 ?s}->>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?	 !6	7	7 C31S]BBBBBC C C C C C C C C C C C C C C	 !2	3	3 Ns<<<MMMMMN N N N N N N N N N N N N N N	 !8	9	9 S
 
 
 ]                   
!!8	9	9 S
 
 
 ]                   
!!C	D	D 	
 
 
 ]    	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 
!!3	4	4 =l+s}<<<<<= = = = = = = = = = = = = = =	 !?	@	@ gCUUUY\Yffffffg g g g g g g g g g g g g g g g g gs   ,00A##A'*A'=BB!B4CCC+DDD"D??EEE66E:=E:F--F14F1G$$G(+G(>HH #H c                  v    t          d          5 } g d| j        k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nz)compute-b24-[1-3,5-9],compute-b25-[1,4,8])zcompute-b24-1zcompute-b24-2zcompute-b24-3zcompute-b24-5zcompute-b24-6zcompute-b24-7zcompute-b24-8zcompute-b24-9zcompute-b25-1zcompute-b25-4zcompute-b25-8rS  rU  s    r   )test_slurm_node_list_online_documentationrX    s    	 !L	M	M QT
 
 
 ]                     s   .22c                     t          j        t          j                  5  t	          d          5 } t          | j                   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          j        t          j                  5  t	          d          5 } t          | j                   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nzcompute-b2[1-,4]zcompute-b2[1,2,compute-b3])rS   rT   r   SlurmParseExceptionrD  r   rT  rU  s    r   test_slurm_invalid_parser[    s   	u0	1	1 ! !$%788 	!C#-   	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!! ! ! ! ! ! ! ! ! ! ! ! ! ! ! 
u0	1	1 ! !$%ABB 	!c#-   	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !sj   A'AA'A	A'A	A''A+.A+C C5CC	CC		CC Cc                  ~    t          d          5 } | j        g| j        k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   )rD  hostnamerT  rU  s    r   test_slurm_missing_node_listr^    s    	 	$	$ /~...../ / / / / / / / / / / / / / / / / /s   266weird_tmp_pathc                 x   d| j         v rt          j        d           t                      5  t	          j        |           }|                    t          j        ddd          }d d d            n# 1 swxY w Y   |j	        j
                            d           |j	        j                            d           i }|j	        j                                                                        D ]}|                    d	          sd
|vr|t#          d	          d                                                              d
d          \  }}|                    d|j                                      dd          ||<   t-          j        d|d         z   d           t-          j        d|d         z   d           d S )Nr   z0test doesn't support newline in 'weird_tmp_path'rF   r   r   rH   rI   r   z#SBATCH=r   z%t0zls z--outputT)shellr   )r  rS   skipr4   r   rO   rP   r	   rQ   r   r    r   r!   r~   r   r   
startswithlenrs   r{   r   r   
subprocess
check_call)r_  r[   r   sbatch_argsrt   keyvals          r   test_slurm_weird_dirrl    s   ~"""FGGG	 D D&n===ooi2AqoCCD D D D D D D D D D D D D D D
 I###I### KY&0022==?? L L||I&& 	a<<S^^%%&,,..44S!<<S;;tSZ88@@sKKC %+j"99FFFF%+i"88EEEEEEs   9A11A58A5paramsr   c                    t                      5  t          j        |          } |j        di | dddii |                    t
          j        ddd          }d d d            n# 1 swxY w Y   |j        j        	                                }d	 |
                                D             }t          |          dk    sJ d
|             d S )NrF   slurm_additional_parametersmem_per_gpurM   r   r   rH   rI   c                     g | ]}d |v |	S )z#SBATCH --memre   r  s     r   r   z+test_slurm_through_auto.<locals>.<listcomp>  s"    FFFqA1E1E1E1E1Er   zUnexpected lines: re   )r4   r   r   ru   rP   r	   rQ   r   r~   r   r   rf  )rm  r5   r[   r   text	mem_liness         r   test_slurm_through_autort    s.   	 D D(999""]]V]]-Y[I\]]]]ooi2AqoCCD D D D D D D D D D D D D D D 9$..00DFFDOO--FFFIy>>Q @Y @ @s   AA''A+.A+c                 r   d }t                      5 }t          j        |           }|                    |          }t	          |d           d|_        t          j        t          j	        d          5  |
                                 d d d            n# 1 swxY w Y   |                    dd	           |                    |          }d|_        t          j        t          j	        d
          5  |
                                 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nc                       t          d          )NzToo bad)r   re   r   r   fail_silentlyz/test_slurm_job_no_stderr.<locals>.fail_silently  s    ###r   rF   zjob is running ...
)r   r   zjob is running ...r   13r@   z!No output/error stream produced !)r4   r   rO   rP   r)   rR   rS   rT   r
   rU   rV   r;   )r5   rw  r3   r[   r   s        r   test_slurm_job_no_stderrry    s   $ $ $ 
 *4&h777oom,,$:;;;;!"]54<PQQQ 	* 	*'')))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	4+++oom,,!" ]54<_``` 	* 	*'')))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	** * * * * * * * * * * * * * * * * *s[   A#D,5B
D,B	D,B	AD,3DD,D	D,D	D,,D03D0)r   r   )r   N)NF)G
contextlibrY   r   rg  typingtppathlibr   unittest.mockr   rS   r   r   r   corer   r   r	   r
   	core.corer   r   Anyr   r)   contextmanagerIteratorr-   r4   r>   rD   r\   markparametrizeboolr   r   Optionalrw   r   r   r   r   r   r   r   r   r   r   r   r  r  ListTupleUnionr$  r(  r+  r7  r<  r>  r   rD  rV  rX  r[  r^  rl  Dictrt  ry  re   r   r   <module>r     sd       				                              @ @ @ @ @ @ @ @ @ @ @ @             RV c  T     'bk)"<= ' ' ' '& & & & & &G G G GfD fT f f f f. -88 7t  7t  7  7  7  7 98 7F;d ;t ; ; ; ; 
7 
7bk#. 
74 
7 
7 
7 
7C O$A    '0D '0 '0 '0 '0 '0T0 0T 0 0 0 0<+t + + + + + 1 1 1 1:) ) ) )
# # # #
<T <d < < < <.4 .D . . . .^ ^ ^ ^ ]]] ;s ;3 ;4 ; ; ; ; 	w	'	%	!E!E!EF	gY	 	JS JBGBHRXc3h=OQT=T4U,V J[_ J J J	 	J
 24`a  s t   	 
R R R R
+ + + +$" " " "
1 1 1 1 #s #r{5;T/U # # # #*g *g *g *gZ   "! ! ! !/ / / /
F F$ F F F F6 B4(8#9::ABGCH$5 A A$ A A A ;:A*t * * * * * * *r   