
    n i8                     r    d dl Z d dlZd dlmZmZ  G d de j                  Z G d dej                  Z	dS )    N)common_spec
model_specc                       e Zd ZdZdZdZdZdS )RotaryScalingTypezRoPE scaling type.r         N)__name__
__module____qualname____doc__LinearSuLlama3     `/var/www/html/auto_sub_bot/venv/lib/python3.11/site-packages/ctranslate2/specs/attention_spec.pyr   r   	   s#        F	
BFFFr   r   c                   4    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZdS )	MultiHeadAttentionSpecFNTr   '  r   c                    t           j        | _        t          j        |          | _        d t          |rdnd          D             | _        |rd | _        d | _	        |rd | _
        d | _        |rd | _        d | _        d | _        |dk    r,t          j        d                              |          | _        |dk    r,t          j        d                              |          | _        |t          j        d                              |          | _        || _        t          j        d                              |
          | _        |,t          j        d                              |          | _        |t0          j        u r-t          j        d                              |	          | _        n9|t0          j        u rd | _        d | _        n|t0          j        u rd | _        d | _         |,t          j        d                              |          | _!        |,t          j        d                              |          | _"        |.t          j        d                              |          | _#        d S d S )	N)rms_normc                 4    g | ]}t          j                    S r   )r   
LinearSpec).0_s     r   
<listcomp>z3MultiHeadAttentionSpec.__init__.<locals>.<listcomp>'   s.     
 
 
)*K"$$
 
 
r   r      r   int32float32int8)$r   OPTIONALqueries_scaler   LayerNormSpec
layer_normrangelinearrelative_position_keysrelative_position_valuesrelative_attention_biasrelative_attention_max_distance!relative_asymmetric_position_keysrelative_left_max_positionrelative_right_max_positionnpdtypetype original_max_position_embeddingsmax_position_embeddings
rotary_dimrotary_interleaverotary_baserotary_scaling_typer   r   rotary_scaling_factorr   rotary_scaling_long_factorrotary_scaling_short_factorr   rotary_low_freq_factorrotary_high_freq_factornum_heads_kvhead_dimsliding_window)selfself_attentionrelative_positionrelative_asymmetric_positionr)   r   r3   r4   r6   r7   r5   r1   r2   r<   r=   r>   s                   r   __init__zMultiHeadAttentionSpec.__init__   sx   $ (0%3XFFF
 
.34NAAQ.O.O
 
 
  	1*.D',0D)" 	8+/D(37D0' 	459D2.2D+/3D,+q0046HW4E4E4J4J05 5D1 #a''+-8G+<+<+A+A', ,D( ! hw//44Z@@DO%6D"!x	2277DDD".+-8F+;+;+@+@AT+U+U("&7&>>>-/Xi-@-@-E-E). .** %(9(<<<26/3700$(9(@@@.2+/3,# " 1 1 6 6| D DDHW--228<<DM%"$(7"3"3"8"8"H"HD &%r   )FFFFFNTNr   r   r   r   NNN)r	   r
   r   rC   r   r   r   r   r      sc         %* % )* !!HI HI HI HI HI HIr   r   )
enumnumpyr.   ctranslate2.specsr   r   IntEnumr   	LayerSpecr   r   r   r   <module>rI      s         5 5 5 5 5 5 5 5       II II II II IIZ1 II II II II IIr   