
    VhA                         d Z ddlZddlmZ ddlmZ ddlmZmZ ddl	Z	ddl
mc mZ ddlmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZ  ej        e          Z d Z! G d de"          Z#dS )zMain training loop.    N)get_xp)write_and_rename)LogProgressbold   )augmentdistribstates
pretrained)apply_model)ModelEMA)evaluatenew_sdr)svd_penalty)pull_metricEMAc                 d    d                     d |                                 D                       S )N | c              3   N   K   | ] \  }}|                                  d | V  !dS )=N)
capitalize).0keyvals      S/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/demucs/solver.py	<genexpr>z_summary.<locals>.<genexpr>   s>      RRhc3))11C11RRRRRR    )joinitems)metricss    r   _summaryr!      s+    ::RR'--//RRRRRRr   c                   N    e Zd Zd Zd Zd ZdedefdZdedefdZd Z	dd
Z
dS )Solverc           	         || _         || _        || _        || _        t	          j        | j        |j        | j                  | _        t          j	        |          | _
        t          t          | j                                                            j        | _        g g d| _        | j                                        D ]b}t#          |j        |          }|dk    r| j        nd}|r:|D ]7}| j        |                             t)          | j        ||                     8ct+          j        t/          |j        j        |j        j        z            |j        j                  g}	|j        j        r)|	t+          j                    t+          j                    gz  }	dD ]^}
t#          |j        |
          }|j        r@|	                     t#          t*          |
                                           di |           _tC          j"        j#        |	 | _        tI                      }|j%        | _%        |j%        dz  | _&        |j%        dz  | _'        tP          )                    d	| j&        *                                           d | _+        d
| _,        |j-        | _-        | j-        j.        | _.        | /                                 d S )N)batchepochr%   cpu)device)shiftsame)scaleremixcheckpoint.thzbest.thzCheckpoint will be saved to %sF )0argsloadersmodel	optimizerr
   get_quantizerquant	quantizerr	   wrapdmodelnextiter
parametersr(   emaskeysgetattremaappendr   r   Shiftintdset
samplerater)   
shift_sameflipFlipChannelsFlipSignprobar   torchnn
Sequentialr   foldercheckpoint_file	best_fileloggerdebugresolve
best_statebest_changedlinkhistory_reset)selfr0   r1   r2   r/   kinddecaysr(   decayaugmentsaugkwxps                r   __init__zSolver.__init__   s|   	
"-dj$*dnUUl5))4
 5 5 7 78899@
 !2..	INN$$ 	W 	WDTXt,,F$(GOOT[[F W# W WEIdO**8DJf+U+U+UVVVV MDI,@49?,R(S(S'+|'>@ @ @ A< 	E-//1A1C1CDDH% 	J 	JCs++Bx J B1A1A B B H HR H HIIIx*H5XXi!y?:Y.5t7K7S7S7U7UVVV!G	y(r   c                    i }| j                                         |d<   | j                                        |d<   | j        |d<   | j        |d<   | j        |d<   | j                                        D ]7\  }}t          |          D ]"\  }}|                                |d| d| <   #8t          | j
                  5 }t          j        ||           d d d            n# 1 swxY w Y   | j        j        }|rj|dz   |z  d	k    r^|dz   | j        j        k    rKt          | j        d
|dz    dz            5 }t          j        ||           d d d            n# 1 swxY w Y   | j        rst          | j                  5 }t%          j        | j         | j                  }| j        |d<   t          j        ||           d d d            n# 1 swxY w Y   d| _        d S d S )Nstater2   rU   rR   r/   ema__r   r   checkpoint_z.thF)r1   
state_dictr2   rU   rR   r/   r;   r   	enumerater   rM   rI   save
save_everyepochsrL   rS   rN   r
   serialize_model)	rW   r&   packagerX   r;   kr>   tmprh   s	            r   
_serializezSolver._serializeM   s   :0022#~88::!\	 $))//++ 	> 	>JD$#D// > >3-0^^-=-=)t))a))**>d233 	%sJw$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% Y)
 	)519
2a77EAIIY<Y<Y!$+0Leai0L0L0L"LMM )QT
7C((() ) ) ) ) ) ) ) ) ) ) ) ) ) )  	&!$.11 )S 0TYGG#'? 
7C((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) !&D	& 	&s6   >C  C$'C$6EEE>?G		GGc           	         | j                                         rt                              d| j                     t	          j        | j         d          }| j                            |d                    | j                            |d                    |d         | j	        dd<   |d         | _
        | j                                        D ];\  }}t          |          D ]&\  }}|                    |d| d	|                     '<dS | j        j        rXt!          j        | j        j        | j        j        
          }| j                            |                                           dS | j        j        rd}| j        j        }|t/          | j        j                  z  |z  }	t                              d|	           t	          j        |	d          }|d         | _
        | j        j        r#| j                            |d         d           n"| j                            |d         d           | j        j        r$| j                            |d                    dS dS dS )z8Reset state of the solver, potentially using checkpoint.zLoading checkpoint model: r'   ra   r2   rU   NrR   rb   rc   )namerepor-   zLoading from %sF)strict)rM   existsrO   inforI   loadr1   load_state_dictr2   rU   rR   r;   r   rf   r/   continue_pretrainedr   	get_modelpretrained_repore   continue_fromrL   parentstrcontinue_bestcontinue_opt)
rW   rk   rX   r;   rl   r>   r1   rp   rootcfs
             r   rV   zSolver._resetg   s   &&(( 	EKKKT5IKKLLLj!5u==GJ&&ww'7888N**7;+?@@@%i0DLO%l3DO"ioo// D D
d'oo D DFAs''0At0A0Aa0A0A(BCCCCDD D Y* 	E(Y2Y.0 0 0E J&&u'7'7'9'9:::::Y$ 	E"D;%DDI3444t;BKK)2...jU++G%l3DOy& K
**7<+@*OOOO
**77+;E*JJJy% E..w{/CDDDDD	E 	EE Er   r    returnc                    t          |d         d          t          |d         d          d}d|v rt          |d         d          |d<   | j        t          |d         d	          |d<   d
|v rt          |d
         d          |d
<   d|v rt          |d         d          |d<   d|v r|d         |d<   d|v rt          |d         d          |d<   d|v rt          |d         d          |d<   |S )z#Formatting for train/valid metrics.lossz.4freco)r   r   nsdr.3fNmsz.2fgradbestbnamepenaltyhloss)formatr5   )rW   r    lossess      r   _format_trainzSolver._format_train   s    76?E2276?E22
 
 W#GFOU;;F6N>%!'$-77F4LW#GFOU;;F6NW#GFOU;;F6Ng%g.F7O &wy'95 A AF9g$WW%5u==F7Or   c                 $   i }d|v rt          |d         d          |d<   d|v rt          |d         d          |d<   | j        j        D ]F}d| }||v rt          ||         d          ||<   d| }||v rt          ||         d          ||<   G|S )zFormatting for test metrics.sdrr   r   sdr_nsdr_)r   r1   sources)rW   r    r   sourcer   s        r   _format_testzSolver._format_test   s    G"75>599F5MW#GFOU;;F6Nj( 	: 	:F!//Cg~~$WS\599s"&""Cg~~$WS\599sr   c                 v   | j         rt                              d           t          | j                   D ]\  }}|                     |d                   }t                              t          d|dz    dt          |                                |                     |d                   }t                              t          d|dz    dt          |                                d|v rZ|                     |d                   }|r=t                              t          d	|dz    dt          |                                d
}t          t          | j                   | j
        j                  D ]=}| j                                         i }t                              d           t                              d           |                     |          |d<   |                     |d                   }t                              t          d|dz    dt          |                                t                              d           t                              d           | j                                         t!          j                    5  |                     |d          }|}d}t%          j        | j                                                  }i |d<   ||d         d<   | j
        j        j        }| j                                        D ]\  }	}
t          |
          D ]\  }}|                                5  |                     |d          }d d d            n# 1 swxY w Y   d|	 d| }||d         |<   ||         }||         }|                    d          r| }| }||k     r|}|j        }|}|d                             |           ||d         d<   	 d d d            n# 1 swxY w Y   |d         |         }t;          | j        j         d|           |gz   }|                    d          rt?          |          }ntA          |          }||d         d<   | j
        j!        j"        d
k    rxtG          | j
        j!                  }|$                    d           t!          j                    5  tK          | j        fddi|}d d d            n# 1 swxY w Y   ||d         d<   |                     |d                   }t                              t          d|dz    dt          |                                ||k    s| j
        j&        j'        rHt                              t          d          |           t%          j        |          | _(        d| _)        |dz   | j
        j        j*        z  d
k    }|| j
        j        dz
  k    }|s|rTt                              d           t                              d           | j
        j        j+        r| j(        }n+t%          j        | j                                                  }| j
        j        j,        o|}t%          j-        | j        |          5  t!          j                    5  t]          | |          |d<   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |                     |d                   }t                              t          d	|dz    dt          |                                | j        /                    |           t`          j1        d
k    rG| 2                    |           t          3                    d| j4        5                                           |r d S ?d S )Nz#Replaying metrics from previous runtrainzTrain Summary | Epoch r   r   validzValid Summary | Epoch testzTest Summary | Epoch r   zF----------------------------------------------------------------------zTraining...zCross validation...F)r   mainrb   rc   r   r   zvalid.r   r   exactTzNew best valid loss %.4fzEvaluating on the test set...)compute_sdrzCheckpoint saved to %s)6rU   rO   rt   rf   r   r   r!   r   rangelenr/   ri   r1   r   _run_one_epochevalrI   no_gradr
   
copy_statere   r   metricr;   r   swap
startswithra   updater   rT   maxminsvdr   dictpopr   rB   train_validrR   rS   everyr   r   
swap_stater   push_metricsr	   rankrn   rP   rM   rQ   )rW   r&   r    	formattedr   bvalidr   ra   r   rX   r;   rl   r>   rp   ab
valid_lossmets	best_lossr]   r   should_evalis_lastr   s                           r   r   zSolver.train   s   < 	?KK=>>>'55 
	c 
	cNE7**77+;<<IKKQeaiQQHY<O<OQQRRT T T**77+;<<IKKQeaiQQHY<O<OQQRRT T T   --gfo>>	 cKK%`UQY%`%`8T]K^K^%`%` a abbb3t|,,di.>?? d	 d	EJGKK!!!KK&&&#22599GG**77+;<<IKKQeaiQQHY<O<OQQRRT T T KK!!!KK-...JOO 6 6++E+??)$**?*?*A*ABB#% +0 (in+"&)//"3"3 6 6JD$"+D// ) )3 XXZZ L L$($7$7U$7$K$KEL L L L L L L L L L L L L L L0d00Q0016(.!#J"3K>>&11 #!"A!"Aq55%*F$'IE$(EG$++F33305GG$W--!66 6 6 6 6 6 6 6 6 6 6 6 6 6 64 !)#.Jty0.3..AAZLPD~~f%% &II		II	'0GGV$y}$q(($)-((y!!!]__ H H)$*GGDGBGGGH H H H H H H H H H H H H H H.5 +**77+;<<IKKQeaiQQHY<O<OQQRRT T T Y&&$).*D&D!;<<jIII"("3E":":$(! !19	(<<AKty/!33G  _g _H%%%;<<<9>& G OEE"-dj.C.C.E.EFFE"in0<W&tz599 R R R R*24[*Q*Q*QR R R R R R R R R R R R R R RR R R R R R R R R R R R R R R !--gfo>>	D!\!\!\xPYGZGZ!\!\]]^^^I""7+++|q  &&&5t7K7S7S7U7UVVV Gd	 d	s   B.O=
M."O=.M22O=5M26A:O==P	P	S--S1	4S1	[&[;[[[[[["	%["	Tc                    | j         }|r| j        d         n| j        d         }t          j        dk    r|r|j                            |           ddg|         }|d|dz    z   }t          |          }|j        rt          ||j                  }t          t          ||| j         j        j        |          }t                      }	t          |          D ]\  }
}|                    | j                  }|r,|                     |          }|                    d          }n|d d d	f         }|d d dd f         }|s4| j         j        r(t)          | j        || j         j        j        d	
          }n|                     |          }|r0t3          | j        d          r| j                            ||          }|j        |j        k    sJ |j        |j        f            t9          t;          d|                                                    }|j        j         dk    rBtC          j"        ||d          }|#                    |          #                    d	          }|}ns|j        j         dk    rGtC          j$        ||d          }|#                    |          }|dz  }|#                    d	          }ntK          d| j         j                    tM          j'        |j(                                      |          }||z                                  |                                z  }d	}| j)        | j)        *                                }|j+        j,        r||j+        j,        |z  z  }i }||z                                  |                                z  |d<   ||d<   |st[          ||.                                          #                    d	          }d	}t_          | j        j0        ||          D ]\  }}}||d| <   |||z  z  }||                                z  |d<   |rb|j1        j2        d	k    rRtg          |j1                  }|4                    d           tk          | j        fi |}||d<   ||j1        j2        |z  z  }||d<   t          | j        j0                  D ]\  }}||         |d| <   |rt|6                                 d	}g }| j        7                                D ]L}|j8        C||j8        j9        :                                dz  z  }|;                    |j8        j9                   M|dz  |d<   |j        j<        rFtL          j=        j>        ?                    | j        7                                |j        j<                   | j         j@        dk    r6| j        A                                D ]\  }}|j8        t          d|           | jC        D                                 | jC        E                                 | jF        d         D ]}|G                                  |	|          }| H                    |          }  |jG        d i |  ~~~~|j        |
k    r n$| j         jI        r|r n| j         j@        dk    r n|r$| jF        d         D ]}|G                                 t          jJ        ||
dz             S )!Nr   r   r   ValidTrainz	 | Epoch )totalupdatesrp   )dimr   )splitoverlaptransform_target   l1none)	reductionmseg      ?zInvalid loss r   r   r   r   r   r   reco_r   unszno gradr%   rP   r&   r.   )Kr/   r0   r	   
world_sizesampler	set_epochr   max_batchesr   r   rO   misc
num_printsr   rf   tor(   r   sumvalid_applyr   r1   r   r   r7   hasattrr   shapetupler   r   optimr   Fl1_lossmeanmse_loss
ValueErrorrI   tensorweightsr5   
model_sizer4   diffqr   detachzipr   r   r   r   r   r   backwardr:   r   datanormr?   	clip_gradrJ   utilsclip_grad_norm_flagnamed_parametersprintr2   step	zero_gradr;   r   r   rP   average)!rW   r&   r   r/   data_loaderlabelrp   r   logprogaverageridxr   mixestimatedimsr   r   r   r   r   nsdrsr   r   wr]   r   rl   	grad_normgradspnr>   logss!                                    r   r   zSolver._run_one_epoch#  s   y/4Odl7++$,w:O!!e!))%000'"5).519...K   	1t/00Efk&*in&?dL L L55%g.. _	 _	LCjj--G ),,w//kkak((aaadm!!!!QRR%. ,TY2 ,&tz3din>R\]^^^;;s++ D-?@@ D*55c7CC>W]222X^W]4S222q'++--0011Dz$&&y7fEEEyy++A..E))z(GvFFFyySyyy|| !A!A!ABBBl4<0033G<<G7N''))GKKMM9DB~)^..00z .
(2--F"Wn1133gkkmmCF6NF4L 7):):;;@@CC'*4:+=ug'N'N & &OFD!/3F+6++,QX%EE!&!6v 3)A--$(^^y!!!%dj77B77$+y!(722!F6N&tz'9:: 3 3	6+/7'v''((  !	..00 2 2Av)!QV[%5%5%7%7%::	QV[111!*c!1v:' .HN22
--//
,. . . 9>U** $
 ; ; = = 0 016>!)Q///##%%%((***9W- ! !CJJLLLLXf%%F%%f--DGN""T"""hb3&&y 5 y~(( ) 	y)  

vsQw///r   N)T)__name__
__module____qualname__r_   rn   rV   r   r   r   r   r   r.   r   r   r#   r#      s        , , ,\& & &4E E E>T d    ,D T     u u unr0 r0 r0 r0 r0 r0r   r#   )$__doc__loggingdorar   
dora.utilsr   dora.logr   r   rI   torch.nn.functionalrJ   
functionalr    r   r	   r
   r   applyr   r>   r   r   r   r   r   r   r   r   	getLoggerr   rO   r!   objectr#   r.   r   r   <module>r     sb            ' ' ' ' ' ' & & & & & & & &           2 2 2 2 2 2 2 2 2 2 2 2             ' ' ' ' ' ' ' '       # # # # # # # #		8	$	$S S Sw0 w0 w0 w0 w0V w0 w0 w0 w0 w0r   