
    Vhxj                        d dl Z d dlZd dlZd dlmZ d dlmc mZ d dl	Z
d dlZd dlmZ 	 d,dededefdZd-d	Z	 	 	 	 	 d.dedededededededededefdZd Zd Zd Z G d dej                  Z G d dej                  Z G d dej                  Z G d d ej                  Z G d! d"ej                  Z G d# d$ej                  Z  G d% d&ej                  Z!d' Z"d( Z#d) Z$d/d+Z%dS )0    N)	rearrangecpu'  lengthdimshiftc                 n   |dz  dk    sJ |t          j        | |                              ddd          z   }|dz  }t          j        |dz  |                              ddd          }||||dz
  z  z  z  }t          j        t          j        |          t          j        |          gd          S )N   r   device   r   )torcharangeviewcatcossin)	r   r   r   r   
max_periodposhalf_dimadimphases	            X/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/demucs/transformer.pycreate_sin_embeddingr      s     7a<<<<
%,vf555::2q!DD
DCaxH<q00055aB??D:$(Q,"789E9IeIe	
        c                    | dz  dk    r"t          d                    |                     t          j        | ||          }t	          | dz            } t          j        t          j        d| d          t          j        |          | z   z            }t          j        d|          	                    d          }t          j        d|          	                    d          }t          j
        ||z                                dd          	                    d                              d|d          |d| dddddf<   t          j        ||z                                dd          	                    d                              d|d          |d| dddddf<   t          j
        ||z                                dd          	                    d                              dd|          || ddddddf<   t          j        ||z                                dd          	                    d                              dd|          || dz   ddddddf<   |dddf                             |          S )z
    :param d_model: dimension of the model
    :param height: height of the positions
    :param width: width of the positions
    :return: d_model*height*width position matrix
       r   zHCannot use sin/cos positional encoding with odd dimension (got dim={:d})r
           r   N)
ValueErrorformatr   zerosintexpr   mathlog	unsqueezer   	transposerepeatr   to)	d_modelheightwidthr   r   pediv_termpos_wpos_hs	            r   create_2d_sin_embeddingr3   %   s    {a++16'??
 
 	
 
Wfe	,	,B'A+GyS'1%%$(:*>*>*H(II H Le$$..q11ELf%%//22E	%("##--a33==a@@GG6STUU q{AAAqqq 		%("##--a33==a@@GG6STUU q{AAAqqq 		%("##--a33==a@@GG1eTT wzz111aaa 		%("##--a33==a@@GG1eTT w{Q111 dAAAg;>>&!!!r   r          ?     @
batch_sizemean_normalizeaugmentmax_global_shiftmax_local_shift	max_scaler   r   c
                    |dz  dk    sJ dt          j        |                               ddd          z  }
|
                    d|d          }
|r|
t          j        |
dd          z  }
|rt
          j                            | |
 d|dg          }t
          j                            | |
 | |dg          }t
          j                            t          j        |           t          j        |          
 d|dg          }|
|z   |z   t          j	        |          z  }
|

                    |          }
|dz  }t          j        |dz  |	                              ddd          }|
|	||dz
  z  z  z  }t          j        t          j        |          t          j        |          gd
                                          S )Nr
   r   r4   r   r   T)r   keepdim)sizer   r   )r   r   r   r*   nanmeannprandomuniformr'   r%   r+   r   r   r   float)r   r   r6   r7   r8   r9   r:   r;   r   r   r   deltadelta_locallog_lambdasr   r   r   s                    r   create_sin_embedding_caperG   I   s    7a<<<<
V$$))"a33
3C
**Q
A
&
&C 7u}Sa6666 
@	!! 00:q7I " 
 
 i''.fj!5L ( 
 
 i''VI	!2!2 2!Z9K ( 
 
 U{[(BF;,?,??
&&..CaxH<q00055aB??D:$(Q,"789E9IeIe	
    eggr   c                 J    t          j        |           }||d d d f         k    S N)r   r   )r   r   s     r   get_causal_maskrJ   v   s(    
,v

CQQQWr   c                     |dv sJ |dk    rLt          j        || t           j                  }d|ddd|f<   t          ||z  | z            }	d|d|	ddf<   |dk    rt          j        || t           j                  }t          j        |          dddf         }
| |z  |
z  t          j        | |dz             z                                                       d| dz
            }|                    d|t          j        dt           j                  	                    |                     n|d	k    rMt          j        |d
z   | d
z   t           j                  }t          j        |d
z             dddf         }
t          j        dt          d
| z  dz  dz                       }||dz   z  d
z                                  }t          j
        |                    d          dd          |g          }| |z  |
z  |z                                                       d| dz             }|                    d|t          j        dt           j                  	                    |                     |ddddf         }nb|dk    r\t          j        |          }|                    |           t          j        | |z  ||                              ||           |k    }|                    |          }|S )zn
    When the input of the Decoder has length T1 and the output T2
    The mask matrix has shape (T2, T1)
    )diagjmaskrA   globalrN   )dtypeTNrL   r   r   rM   r
         ?r   rA   r   )	generatorr   )r   r#   boolr$   r   longclampscatter_ones	expand_asr   flip	Generatormanual_seedrandreshaper+   )T1T2	mask_typesparse_attn_windowglobal_windowmask_random_seedsparsityr   maskline_windowrowscolstgenes                 r   get_elementary_maskrj   {   s    =====H{2r444"&QQQ-",r122 $\k\111_F{2r444|B4("Wt^el,>+>@RUV@VWWWTVVU1b1f 	
 	auz!5:>>>HHNNOOOO	g		{2626<<<|BF##AAAtG,LCRC! 34455!a%[1_!!##Iq		#2#*++R$"((**00BF;;auz!5:>>>HHNNOOOAbD!B$J	h		f---)***JrBw$v>>>FFr2NN 	
 776??DKr   c                      ddl m} |                    d          }	 fd|	D             }
t          j        |
                              d          dk    }|                    |d                   S )z
    Return a SparseCSRTensor mask that is a combination of elementary masks
    mask_type can be a combination of multiple masks: for instance "diag_jmask_random"
    r   )SparseCSRTensor_c                 <    g | ]}t          |          S  )rj   )	.0rd   r]   r^   r   ra   rb   r`   rc   s	     r   
<listcomp>zget_mask.<locals>.<listcomp>   sM         			
 		
  r   )axisN)xformers.sparserl   splitr   stacksum
from_dense)r]   r^   r_   r`   ra   rb   rc   r   rl   
mask_types	all_masks
final_masks   `` `````    r   get_maskr{      s     0/////%%J            I Y''+++33a7J%%j&6777r   c            	       R     e Zd Z	 	 d
dedededef fdZed             Zd	 Z xZ	S )ScaledEmbeddingr4         @num_embeddingsembedding_dimscaleboostc                     t                                                       t          j        ||          | _        | j        j        xj        ||z  z  c_        || _        d S rI   )super__init__nn	Embedding	embeddingweightdatar   )selfr   r   r   r   	__class__s        r   r   zScaledEmbedding.__init__   sV     	nmDD""eem3""


r   c                 *    | j         j        | j        z  S rI   )r   r   r   )r   s    r   r   zScaledEmbedding.weight   s    ~$tz11r   c                 <    |                      |          | j        z  S rI   )r   r   r   xs     r   forwardzScaledEmbedding.forward   s    ~~a  4:--r   )r4   r~   )
__name__
__module____qualname__r$   rC   r   propertyr   r   __classcell__r   s   @r   r}   r}      s        
 
 

 
 	

 
 
 
 
 
 
 2 2 X2. . . . . . .r   r}   c                   4     e Zd ZdZddedef fdZd Z xZS )	
LayerScalezLayer scale from [Touvron et al 2021] (https://arxiv.org/pdf/2103.17239.pdf).
    This rescales diagonaly residual outputs close to 0 initially, then learnt.
    r   Fchannelsinitc                     t                                                       || _        t          j        t          j        |d                    | _        || j        j        dd<   dS )z
        channel_last = False corresponds to (B, C, T) tensors
        channel_last = True corresponds to (T, B, C) tensors
        T)requires_gradN)	r   r   channel_lastr   	Parameterr   r#   r   r   )r   r   r   r   r   s       r   r   zLayerScale.__init__   s[    
 	(\%+hd"K"K"KLL
!
r   c                 L    | j         r
| j        |z  S | j        d d d f         |z  S rI   )r   r   r   s     r   r   zLayerScale.forward   s3     	+:>!:aaag&**r   )r   F)	r   r   r   __doc__r$   rC   r   r   r   r   s   @r   r   r      si         " " "E " " " " " "+ + + + + + +r   r   c                   (     e Zd Z fdZ fdZ xZS )MyGroupNormc                 :     t                      j        |i | d S rI   )r   r   )r   argskwargsr   s      r   r   zMyGroupNorm.__init__  s%    $)&)))))r   c                     |                     dd          }t                                          |                               dd          S )zh
        x: (B, T, C)
        if num_groups=1: Normalisation on all T and C together for each B
        r   r
   )r)   r   r   )r   r   r   s     r   r   zMyGroupNorm.forward  s>    
 KK1wwq!!++Aq111r   r   r   r   r   r   r   r   s   @r   r   r     sQ        * * * * *2 2 2 2 2 2 2 2 2r   r   c                   X     e Zd Zddej        ddddddddddd	d
ddddf fd	ZddZ xZS )MyTransformerEncoderLayer   皙?r   Fh㈵>-C6?NrL   *     2   ffffff?c                    ||d}t                                          ||||||	||||
  
         || _        || _        |r|s|| _        || _        || _        || _        |rDt          t          |          |fd|	i|| _
        t          t          |          |fd|	i|| _        d | _        | j        |z  r#t          t          |          |          | _        |
rt          ||d          nt          j                    | _        |
rt          ||d          nt          j                    | _        |rOt'          |||||r|nd          | _        |                     dt-          j        d	d	                     || _        d S d S )
Nr   rO   )
r,   nheaddim_feedforwarddropout
activationlayer_norm_epsbatch_first
norm_firstr   rO   eps
num_groupsnum_channelsTr   r   r   auto_sparsitysrc_maskr   )r   r   sparser   r_   r`   ra   rc   r   r$   norm1norm2norm_outr   r   r   Identitygamma_1gamma_2MultiheadAttention	self_attn__setattr__r   r#   rb   )r   r,   r   r   r   r   
group_normr   r   r   layer_scaleinit_valuesr   rO   r   r_   rb   r`   ra   r   rc   r   factory_kwargsr   s                          r   r   z"MyTransformerEncoderLayer.__init__  s   0 %+U;;+!)#! 	 	
 	
 	
 * 	%  3!**<'%2"$DM 	e$S__gdd>dUcddDJ$S__gdd>dUcddDJ?X% 	X'3x==wWWWDM6ATJwT222r{}} 	 7BTJwT222r{}} 	  	5/[*7>hhQ  DN ZQ):):;;;$4D!!!	5 	5r   c           
      L   |j         }|}|j        \  }}}| j        rh| j        sa|J | j        }|j        d         |k    rEt          ||| j        | j        | j        | j	        | j
        |          }|                     d|           | j        r||                     |                     |                     |          ||                    z   }||                     |                     |                     |                              z   }| j        r|                     |          }n~|                     ||                     |                     |||                    z             }|                     ||                     |                     |                    z             }|S )zw
        if batch_first = False, src shape is (T, B, C)
        the case where batch_first=True is not covered
        Nr   r   )r   shaper   r   r   r{   r_   r`   ra   rb   rc   r   r   r   	_sa_blockr   r   	_ff_blockr   r   )	r   srcr   src_key_padding_maskr   r   TBCs	            r   r   z!MyTransformerEncoderLayer.forwardS  s   
 '1a; 	7t1 	7###}H~b!Q&&#N+&)M	 	   X666? 	@DLLtzz!}}h8LMM  A DLL

1!>!>???A} %MM!$$

DLL8=Q!R!RSSS A 

1t||DNN1,=,=>>>??Ar   )NN)r   r   r   Frelur   r   r   r   s   @r   r   r     s        
 6-A5 A5 A5 A5 A5 A5F& & & & & & & &r   r   c                        e Zd Zddej        ddddddddddd	d
ddddfdedededededededededef fdZddZ	ddZ
d Zd Z xZS )CrossTransformerEncoderLayerr   r   r   Fr   rL   r   r   r   r   Nr,   r   r   r   r   r   r   r   r   r   c                    ||d}t                                                       || _        || _        |r|s|| _        || _        || _        || _        |  t          j	        ||||          | _
        t          j        ||fi || _        t          j        |          | _        t          j        ||fi || _        |	| _        |  |  |  |
rgt#          t%          |
          |fd|i|| _        t#          t%          |
          |fd|i|| _        t#          t%          |
          |fd|i|| _        nKt          j        |fd|i|| _        t          j        |fd|i|| _        t          j        |fd|i|| _        d | _        | j        |z  r#t#          t%          |          |          | _        |rt1          ||d          nt          j                    | _        |rt1          ||d          nt          j                    | _        t          j        |          | _        t          j        |          | _        t=          |t>                    r|                      |          | _!        n|| _!        |rQt          |||||r|nd          | _
        |s4| "                    dtG          j$        d	d	                     || _%        d S d S d S )
Nr   )r   r   r   r   Tr   r   rd   r   )&r   r   r   r   r_   r`   ra   rc   r   r   
cross_attnLinearlinear1Dropoutr   linear2r   r   r$   r   r   norm3	LayerNormr   r   r   r   r   dropout1dropout2
isinstancestr_get_activation_fnr   r   r   r#   rb   )r   r,   r   r   r   r   r   r   r   r   r   r   r   r_   rb   r`   ra   rc   r   r   rO   r   r   r   s                          r   r   z%CrossTransformerEncoderLayer.__init__}  s#   0 %+U;;* 	%  3!**<'%2"$DM"/UGF F F y/LL^LLz'**y'LL^LL$ 	U$S__gdd>dUcddDJ$S__gdd>dUcddDJ$S__gdd>dUcddDJJgTT>T^TTDJgTT>T^TTDJgTT>T^TTDJ?X% 	X'3x==wWWWDM 7BTJwT222r{}} 	 7BTJwT222r{}} 	 
7++
7++ j#&& 	)"55jAADOO(DO 	90[*7>hhQ@ @ @DO ! 9  Q):):;;;(8%%%	9 	99 9r   c           
         |j         }|j        \  }}}|j        \  }}}| j        ry| j        sr|J | j        }|j        d         |k    s|j        d         |k    rEt          ||| j        | j        | j        | j	        | j
        |          }|                     d|           | j        r||                     |                     |                     |          |                     |          |                    z   }	|	|                     |                     |                     |	                              z   }	| j        r|                     |	          }	n~|                     ||                     |                     |||                    z             }	|                     |	|                     |                     |	                    z             }	|	S )z
        Args:
            q: tensor of shape (T, B, C)
            k: tensor of shape (S, B, C)
            mask: tensor of shape (T, S)

        Nr   rd   )r   r   r   r   rd   r{   r_   r`   ra   rb   rc   r   r   r   	_ca_blockr   r   r   r   r   r   )
r   qkrd   r   r   r   r   Sr   s
             r   r   z$CrossTransformerEncoderLayer.forward  s    '1a'1a; 	/t1 	/<<<9Dz"~""djn&9&9N+&)M	 	   ...? 	@DLL

1tzz!}}d!S!STTTADLL

1!>!>???A} %MM!$$

1t||DNN1a,F,FGGGHHA

1t||DNN1,=,=>>>??Ar   c                 l    |                      ||||d          d         }|                     |          S )NF)	attn_maskneed_weightsr   )r   r   )r   r   r   r   r   s        r   r   z&CrossTransformerEncoderLayer._ca_block  s4    OOAq!yuOMMaP}}Qr   c           	          |                      |                     |                     |                     |                                        }|                     |          S rI   )r   r   r   r   r   r   s     r   r   z&CrossTransformerEncoderLayer._ff_block  sG    LLdoodll1oo&F&FGGHH}}Qr   c                     |dk    rt           j        S |dk    rt           j        S t          d                    |                    )Nr   geluz&activation should be relu/gelu, not {})r   r   r   RuntimeErrorr"   )r   r   s     r   r   z/CrossTransformerEncoderLayer._get_activation_fn  sC    6M6!!6MCJJ:VVWWWr   rI   )r   r   r   r   r   r$   rC   rR   r   r   r   r   r   r   r   s   @r   r   r   |  sA       
  $6 $!!  -S9 S9S9 S9 	S9
 S9 S9 S9 S9 S9 S9 S9 S9 S9 S9 S9 S9j$ $ $ $N       
     X X X X X X Xr   r   c            ?           e Zd Zddddddddddddd	dd
ddddddg dddddddddfdededededededededededededed ed!ed"ej        e         d#ed$ed%ed&ed'ed(ed)e	d*ed+ed,ed-ed.ed/ed0ed1ef> fd2Z
d3 Zd4 Zd5 Z xZS )6CrossTransformerEncoderr   g      @      Fr    i  Tr5   Nr   r4   )g     @r4   gffffff?rL   r   r   r   r   r   embhidden_scale	num_heads
num_layerscross_firstr   max_positionsnorm_innorm_in_groupr   r   r   r   weight_decaylrr   r   sin_random_shiftweight_pos_embedcape_mean_normalizecape_augmentcape_glob_loc_scalesparse_self_attnsparse_cross_attnr_   rb   r`   ra   r   rc   c                     t                                                       	 ||z  dk    sJ t          ||z            } || _        |rdnd| _        || _        || _        || _        || _        || _	        |dk    r|| _
        || _        || _        |dk    rt          ||d          | _        || _        |rt           j        nt           j        }!|  |  |	r3t'          j        |          | _        t'          j        |          | _        nw|
rEt/          t          |
          |          | _        t/          t          |
          |          | _        n0t'          j                    | _        t'          j                    | _        t'          j                    | _        t'          j                    | _        i d|d|d	| d
|d|!d|d|d|d|d|d|d|d|d|d|dd}"t9          |"          }#|#                    d|i           t9          |"          }$|$                    d|i           t=          |          D ]}%|%dz  | j        k    rI| j                            tA          di |#           | j                            tA          di |#           Y| j                            tC          di |$           | j                            tC          di |$           d S )Nr   r   capescaledg?)r   r,   r   r   r   r   r   r   r   r   r_   rb   r`   ra   rc   r   r   Tr   r
   ro   )"r   r   r$   r   classic_parityr   r   r  r  r  r  r	  r
  r}   position_embeddingsr  r   r   r   r   r   r  	norm_in_tr   r   
ModuleListlayerslayers_tdictupdaterangeappendr   r   )'r   r   r   r   r   r   r   r   r  r  r  r   r   r   r   r  r  r   r   r  r  r  r	  r
  r  r  r_   rb   r`   ra   r   rc   
hidden_dimr   kwargs_commonkwargs_classic_encoderkwargs_cross_encoderidxr   s'                                         r   r   z CrossTransformerEncoder.__init__  s   D 		Y!####|+,,
$ $/5aaA$( 0 0&==':D$ ,D':D$(??'6}cQT'U'U'UD$'+7QVV
! 	+<,,DL\#..DNN 	+&s='9'93??DL(]););SAADNN;==DL[]]DN moo
s
Y
 z
 w	

 *
 *
 *
 
 ;
 
  0
 !"4
 ]
 
 ]
  4!
& "&m!4!4%%&'
 	 	 	  $M22##'%
 	 	 	 $$ 	 	CQw$---""#<#V#V?U#V#VWWW$$-GG0FGG   
 ""#?#W#WBV#W#WXXX$$0HH3GHH   	 	r   c                    |j         \  }}}}t          ||||j        | j                  }t	          |d          }t	          |d          }|                     |          }|| j        |z  z   }|j         \  }}}t	          |d          }|                     ||||j                  }	t	          |	d          }	|                     |          }|| j        |	z  z   }t          | j
                  D ]m}
|
dz  | j        k    r- | j        |
         |          } | j        |
         |          }=|} | j        |
         ||          } | j        |
         ||          }nt	          |d|          }t	          |d          }||fS )Nzb c fr t1 -> b (t1 fr) czb c t2 -> b t2 czt2 b c -> b t2 cr
   zb (t1 fr) c -> b c fr t1)t1zb t2 c -> b c t2)r   r3   r   r   r   r  r  _get_pos_embeddingr  r  r   r  r  r  )r   r   xtr   r   Frr]   
pos_emb_2dr^   pos_embr  old_xs               r   r   zCrossTransformerEncoder.forward  s   w1b",r2qx
 

 z+EFF
a344LLOO%
2281br-..))"aAH==G%788^^B$''11)) 	3 	3CQw$---$DK$Q'''T]3'++$DK$Q++'T]3'E22a3;;;r-.."ur   c                    | j         dk    r6t          j        | j        dz             }t	          ||||| j                  }n| j         dk    rr| j        rJt          ||||| j        | j        | j	        | j
        d         | j
        d         | j
        d         
  
        }nat          ||||| j        | j        d	          }n@| j         d
k    r5t          j        ||          }|                     |          d d d f         }|S )Nr   r   )r   r   r   r  r   r
   )r   r   r7   r8   r9   r:   r;   F)r   r   r7   r8   r  r   )r   rA   	randranger  r   r   trainingrG   r  r	  r
  r   r   r  )r   r   r   r   r   r   r%  r   s           r   r!  z*CrossTransformerEncoder._get_pos_embedding  s1   8u$T%:Q%>??E*1E&T_  GG X} 3!##'#; -%)%=a%@$($<Q$?"6q9   4!##'#;!   X!!,q000C..s33AAAtG<Gr   c                 z    t          |                                           | j        d}| j        
| j        |d<   |S )N)paramsr  r  )list
parametersr  r  )r   groups     r   make_optim_groupz(CrossTransformerEncoder.make_optim_group  s<     1 122DDUVV7'E$Kr   )r   r   r   r$   r   rC   rR   tpOptionalr,  r   r   r!  r/  r   r   s   @r   r   r     s7        !!!# #!!%! !"%$(!$6$6$6!&"' ""%#Aw ww w 	w
 w w w w w w w w w w w  !w" K#w$ %w& 'w( )w*  +w, "-w. /w0 "1w2 3w4  5w6 7w8 9w:  ;w< =w> ?w@ Aw w w w w wr  <# # #J      r   r   c                   @     e Zd Z	 	 	 	 	 	 	 	 d fd	Z	 	 	 	 ddZ xZS )	r   r    TFNc                 P   t                                                       |

J d            || _        t          j                            |||          | _        t          j                            |||          | _        t          j                            |||          | _        t          j        	                    |          | _
        t          j                            |||          | _        t          j        	                    |          | _        |	| _        |
| _        d S )Nzsanity check)bias)r   r   r   r   r   r   r   r   vr   	attn_dropproj	proj_dropr   r   )r   	embed_dimr   r   r4  add_bias_kvadd_zero_attnkdimvdimr   r   r   s              r   r   zMultiheadAttention.__init__  s     	((.((("IDAAIDAAIDAA))'22HOOIy$??	))'22&*r   c                    | j         sE|                    ddd          }|                    ddd          }|                    ddd          }|j        \  }}	}
|j        \  }}}
|                     |                              ||	| j        |
| j        z                                dddd          }|                    dd          }|                     |                              ||| j        |
| j        z                                dddd          }|                    dd          }|                     |                              ||| j        |
| j        z                                dddd          }|                    dd          }| j	        r|J t          |||| j	                  }nt          ||||| j                  }|                    || j        |	|
| j        z            }|                    dd                              ||	|
          }|                     |          }|                     |          }| j         s|                    ddd          }|d fS )Nr   r   r
      )rc   )r   )r   permuter   r   r\   r   flattenr   r5  r   dynamic_sparse_attentionscaled_dot_product_attentionr6  r)   r7  r8  )r   querykeyvaluekey_padding_maskr   r   average_attn_weightsr   N_qr   N_kr   r   r5  r   s                   r   r   zMultiheadAttention.forward  sA     	+MM!Q**E++aA&&CMM!Q**EK	3I	3 FF5MMWQT^Q$.-@AAWQ1a   	

 IIaOOFF3KKWQT^Q$.-@AAWQ1a   	

 IIaOOFF5MMWQT^Q$.-@AAWQ1a   	

 IIaOO 	Y$$$(Aq4;MNNNAA,Q1iXXXAIIaa4>.ABBKK1%%aa00IIaLLNN1 	#		!Q""A$wr   )r    TFFNNFN)NTNTr   r   s   @r   r   r     sy        
 + + + + + +< !1 1 1 1 1 1 1 1r   r   c                     ddl m} | |                    d          dz  z  }  || |                    dd          |          }t          j        j                            |d          }|S )Nr   )masked_matmulr   rP   r   )xformers.opsrL  r>   r)   r   r   
functionalsoftmax)r   r   att_maskrL  atts        r   scaled_query_key_softmaxrR  #  sm    ******	QVVBZZCA
-1;;r2..
9
9C
(

%
%c2
.
.CJr   c                 J    t          | ||          } ||          }||z  }|S )N)rP  )rR  )r   r   r5  rP  r   rQ  ys          r   rC  rC  +  s1    
"1a(
;
;
;C
'#,,CaAHr   c                    t          j        d| |          }t          j        || gd          }|                    d          }|                    ddd                                                                          S )Nzbtf,bfhi->bhtir   r   r   r
   r   )r   einsumr   argmaxr@  byte
contiguous)r   Rqqbucketss       r   _compute_bucketsr]  2  sq    	&1	-	-B	B9"	%	%	%BiiBiG??1a##((**55777r   Tc                 p   ddl m}m} d}d}	d | ||fD             \  } }}t          j                    5  t          j        d| j        d         ||	dz  | j        	          }
t          | |
          }t          ||
          } |||||          \  }}d d d            n# 1 swxY w Y    || |||||          S )
Nr   )find_locations!sparse_memory_efficient_attention    r   c                 6    g | ]}|                                 S ro   )rY  )rp   r   s     r   rq   z,dynamic_sparse_attention.<locals>.<listcomp>?  s     EEEAEEEr   r   r   r
   r   )	rM  r_  r`  r   no_gradrandnr   r   r]  )rD  rE  rF  rc   infer_sparsity	attn_biasr_  r`  n_hashes	proj_sizerZ  bucket_query
bucket_keyrow_offsetscolumn_indicess                  r   rB  rB  :  sC   NNNNNNNNHIEE%e1DEEEE3	 @ @K5;r?Hi1nU\ZZZ'q11%c1--
&4n*h'@ '@#^	@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ -,sE;	C C Cs   ABB!$B!)r   r   r   )r   r   )r    r    r4   r   r5   )TN)&rA   typingr0  r   torch.nnr   torch.nn.functionalrN  r   numpyr@   r&   einopsr   r$   r   r3   rR   rC   r   rG   rJ   rj   r{   Moduler}   r   	GroupNormr   TransformerEncoderLayerr   r   r   r   rR  rC  r]  rB  ro   r   r   <module>ru     s2                                   EJ "%   $!" !" !" !"T " * **	* * 	*
 * * * * * * * * *Z  
4 4 4n"8 "8 "8J. . . . .bi . . .*+ + + + + + + +,
2 
2 
2 
2 
2", 
2 
2 
2j j j j j : j j jZLX LX LX LX LX29 LX LX LXdA A A A Abi A A ANK K K K K K K K\    8 8 8C C C C C Cr   