
    ~Whk                     ^   d Z ddlmc mZ ddlmZ  G d d          Z G d de          Z G d d	e          Z	 G d
 de          Z
 G d de          Z G d de          Z G d de          Z G d de          Z G d deej        j        j                  ZeZe	Ze
ZeZeZeZeZdS )znLegacy v1 optimizer classes.

For more examples see the base class `tf.compat.v1.keras.optimizers.Optimizer`.
    N)backendc                   V    e Zd ZdZd ZdZd Zd Zd Zd Z	d Z
d	 Zed
             ZdS )	Optimizera  Abstract optimizer base class.

    Note: this is the parent class of all optimizers, not an actual optimizer
    that can be used for training models.

    All Keras optimizers support the following keyword arguments:

        clipnorm: float >= 0. Gradients will be clipped
            when their L2 norm exceeds this value.
        clipvalue: float >= 0. Gradients will be clipped
            when their absolute value exceeds this value.
    c                     ddh}|D ]L}||vrt          dt          |          z             ||         dk     rt          d| d||                    M| j                            |           g | _        g | _        d S )Nclipnorm	clipvaluez1Unexpected keyword argument passed to optimizer: r   z	Expected z >= 0, received: )	TypeErrorstr
ValueError__dict__updateupdatesweights)selfkwargsallowed_kwargsks       c/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/keras/optimizers/optimizer_v1.py__init__zOptimizer.__init__)   s    $k2 	N 	NA&&G#a&&P   ay1}} !LQ!L!L!L!LMMM V$$$    Fc                     t           )a   Creates and sets all optimizer weights.

        Args:
          params: list or tuple of `Variable` objects that will be minimized
            using this optimizer.

        Returns:
          Specific weight values that are used in `get_updates`
        NotImplementedError)r   paramss     r   _create_all_weightszOptimizer._create_all_weights;   s
     "!r   c                     t           Nr   r   lossr   s      r   get_updateszOptimizer.get_updatesG       !!r   c                      t          j        ||          }t          d |D                       rt          d          t	           d          r fd|D             }t	           d          r fd|D             }|S )aW  Returns gradients of `loss` with respect to `params`.

        Args:
            loss: Loss tensor.
            params: List of variables.

        Returns:
            List of gradient tensors.

        Raises:
            ValueError: In case any gradient cannot be computed (e.g. if
              gradient function not implemented).
        c              3      K   | ]}|d u V  	d S r    ).0gs     r   	<genexpr>z*Optimizer.get_gradients.<locals>.<genexpr>Y   s&      ((QqDy((((((r   zAn operation has `None` for gradient. Please make sure that all of your ops have a gradient defined (i.e. are differentiable). Common ops without gradient: backend.argmax, backend.round, backend.eval.r   c                 D    g | ]}t          j        |j                  S r$   )tfclip_by_normr   r%   r&   r   s     r   
<listcomp>z+Optimizer.get_gradients.<locals>.<listcomp>b   s'    FFF1R_Q66FFFr   r   c                 R    g | ]#}t          j        |j         j                  $S r$   )r)   clip_by_valuer   r+   s     r   r,   z+Optimizer.get_gradients.<locals>.<listcomp>d   s>         T^OT^DD  r   )r   	gradientsanyr   hasattr)r   r   r   gradss   `   r   get_gradientszOptimizer.get_gradientsJ   s     !$//((%((((( 	?   4$$ 	GFFFFFFFE4%% 	     E r   c                 <   | j         }t          |          t          |          k    rOt          dt          t          |                    z   dz   t          t          |                    z   dz             g }t	          j        |          }t          |||          D ]i\  }}}|j        |j        k    r<t          dt          |j                  z   dz   t          |j                  z             |                    ||f           jt	          j	        |           dS )a  Sets the weights of the optimizer, from Numpy arrays.

        Should only be called after computing the gradients
        (otherwise the optimizer has no weights).

        Args:
            weights: a list of Numpy arrays. The number of arrays and their
              shape must match number of the dimensions of the weights of the
              optimizer (i.e. it should match the output of `get_weights`).

        Raises:
            ValueError: in case of incompatible weight shapes.
        z%Length of the specified weight list (z9) does not match the number of weights of the optimizer ()zOptimizer weight shape z+ not compatible with provided weight shape N)
r   lenr   r
   r   batch_get_valuezipshapeappendbatch_set_value)r   r   r   weight_value_tuplesparam_valuespvpws           r   set_weightszOptimizer.set_weightsj   s5    v;;#g,,&&7c'll##$MN c&kk""# 	   !.v66L&':: 	/ 	/HB1x17"" -"(mm$CD !'ll#    &&1v.... 344444r   c                 4    t          j        | j                  S )zyReturns the current value of the weights of the optimizer.

        Returns:
            A list of numpy arrays.
        )r   r7   r   r   s    r   get_weightszOptimizer.get_weights   s     &t|444r   c                 r    i }t          | d          r
| j        |d<   t          | d          r
| j        |d<   |S )Nr   r   )r1   r   r   r   configs     r   
get_configzOptimizer.get_config   sH    4$$ 	/!%F:4%% 	1"&.F;r   c                      | di |S )Nr$   r$   )clsrG   s     r   from_configzOptimizer.from_config   s    s}}V}}r   N)__name__
__module____qualname____doc__r   _HAS_AGGREGATE_GRADr   r    r3   rA   rD   rH   classmethodrK   r$   r   r   r   r      s              
" 
" 
"" " "  @"5 "5 "5H5 5 5     [  r   r   c                   <     e Zd ZdZ	 d	 fd	Zd Zd Z fdZ xZS )
SGDa  Stochastic gradient descent optimizer.

    Includes support for momentum,
    learning rate decay, and Nesterov momentum.

    Args:
        lr: float >= 0. Learning rate.
        momentum: float >= 0. Parameter that accelerates SGD in the relevant
          direction and dampens oscillations.
        decay: float >= 0. Learning rate decay over each update.
        nesterov: boolean. Whether to apply Nesterov momentum.
    {Gz?        Fc                     t                      j        d	i | t          j        | j        j                  5  t          j        ddd          | _        t          j        |d          | _        t          j        |d          | _	        t          j        |d          | _
        d d d            n# 1 swxY w Y   || _        || _        d S )
Nr   int64
iterationsdtypenamelrr[   momentumdecayr$   )superr   r   
name_scope	__class__rL   variablerX   r\   r^   r_   initial_decaynesterov)r   r\   r^   r_   re   r   rb   s         r   r   zSGD.__init__   s	    	""6""" 788 	? 	?%.|  DO &r555DG#,XJGGGDM )%g>>>DJ	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? #    A.B33B7:B7c                 V    d |D             }d |D             }| j         g|z   | _        |S )Nc                 6    g | ]}t          j        |          S r$   r   	int_shaper%   r?   s     r   r,   z+SGD._create_all_weights.<locals>.<listcomp>   #    7771'#A&&777r   c                 6    g | ]}t          j        |          S r$   r   zerosr%   r9   s     r   r,   z+SGD._create_all_weights.<locals>.<listcomp>   s"    <<<E7=''<<<r   rX   r   )r   r   shapesmomentss       r   r   zSGD._create_all_weights   s>    77777<<V<<<(72r   c           
      >   |                      ||          }t          j        j                            | j        d          g| _        | j        }| j        dk    rB|dd| j	        t          j
        | j        t          j        | j	                            z  z   z  z  }|                     |          }t          |||          D ]\  }}}| j        |z  ||z  z
  }	| j                            t          j        j                            ||	                     | j        r|| j        |	z  z   ||z  z
  }
n||	z   }
t'          |dd           |                    |
          }
| j                            t          j        j                            ||
                     | j        S N   r         ?
constraint)r3   r)   compatv1
assign_addrX   r   r\   rd   r_   castr   rZ   r   r8   r^   r:   assignre   getattrrx   )r   r   r   r2   r\   rs   r?   r&   mvnew_ps              r   r    zSGD.get_updates   s   ""400	//CCDW!!jgdow}TZ/H/HIIJJB **62265'22 	? 	?GAq!!BF*AL	 3 3Aq 9 9:::} DMA--Q6A q,--9U++L	 3 3Au = =>>>>|r   c                    t          t          j        | j                            t          t          j        | j                            t          t          j        | j                            | j        d}t                                                      }t          t          |                                          t          |                                          z             S )N)r\   r^   r_   re   )floatr   	get_valuer\   r^   r_   re   r`   rH   dictlistitemsr   rG   base_configrb   s      r   rH   zSGD.get_config   s    )$'2233g/>>??7,TZ8899	
 
 gg((**D**,,--V\\^^0D0DDEEEr   )rT   rU   rU   F	rL   rM   rN   rO   r   r   r    rH   __classcell__rb   s   @r   rS   rS      s          :?! ! ! ! ! !    @F F F F F F F F Fr   rS   c                   :     e Zd ZdZd
 fd	Zd Zd Z fd	Z xZS )RMSpropa  RMSProp optimizer.

    It is recommended to leave the parameters of this optimizer
    at their default values
    (except the learning rate, which can be freely tuned).

    Args:
      lr: float >= 0. Learning rate.
      rho: float >= 0.
      epsilon: float >= 0. Fuzz factor.
        If `None`, defaults to `backend.epsilon()`.
      decay: float >= 0. Learning rate decay over each update.
    MbP??NrU   c                     t                      j        d	i | t          j        | j        j                  5  t          j        |d          | _        t          j        |d          | _        t          j        |d          | _	        t          j        ddd          | _
        d d d            n# 1 swxY w Y   |t          j                    }|| _        || _        d S )
Nr\   r]   rhor_   r   rW   rX   rY   r$   )r`   r   r   ra   rb   rL   rc   r\   r   r_   rX   epsilonrd   r   r\   r   r   r_   r   rb   s         r   r   zRMSprop.__init__   s   ""6""" 788 	 	&r555DG'%888DH )%g>>>DJ%.|  DO		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ?o''G"rf   c                 ,    d |D             }|| _         |S )Nc                     g | ]<}t          j        t          j        |          t          j        |                     =S )rZ   r   ro   rj   rZ   rk   s     r   r,   z/RMSprop._create_all_weights.<locals>.<listcomp>  sL     
 
 
 M'+A..gmA6F6FGGG
 
 
r   r   )r   r   accumulatorss      r   r   zRMSprop._create_all_weights  s/    
 

 
 
 $r   c           
      |   |                      ||          }|                     |          }t          j        j                            | j        d          g| _        | j        }| j	        dk    rB|dd| j
        t          j        | j        t          j        | j
                            z  z   z  z  }t          |||          D ]\  }}}| j        |z  d| j        z
  t          j        |          z  z   }	| j                            t          j        j                            ||	                     |||z  t          j        |	          | j        z   z  z
  }
t+          |dd           |                    |
          }
| j                            t          j        j                            ||
                     | j        S ru   )r3   r   r)   ry   rz   r{   rX   r   r\   rd   r_   r|   r   rZ   r8   r   squarer:   r}   sqrtr   r~   rx   r   r   r   r2   r   r\   r?   r&   anew_ar   s              r   r    zRMSprop.get_updates  s   ""400//77	//CCDW!!jgdow}TZ/H/HIIJJB 65,77 
	? 
	?GAq!HqLC$(Nbill#BBEL	 3 3Au = =>>>Q',u"5"5"DEEE q,--9U++L	 3 3Au = =>>>>|r   c                    t          t          j        | j                            t          t          j        | j                            t          t          j        | j                            | j        d}t                                                      }t          t          |                                          t          |                                          z             S N)r\   r   r_   r   r   r   r   r\   r   r_   r   r`   rH   r   r   r   r   s      r   rH   zRMSprop.get_config2  s    )$'2233*4844557,TZ8899|	
 
 gg((**D**,,--V\\^^0D0DDEEEr   )r   r   NrU   r   r   s   @r   r   r      s         # # # # # #    :F F F F F F F F Fr   r   c                   :     e Zd ZdZd	 fd	Zd Zd Z fdZ xZS )
Adagrada  Adagrad optimizer.

    Adagrad is an optimizer with parameter-specific learning rates,
    which are adapted relative to how frequently a parameter gets
    updated during training. The more updates a parameter receives,
    the smaller the updates.

    It is recommended to leave the parameters of this optimizer
    at their default values.

    # Arguments
        lr: float >= 0. Initial learning rate.
        epsilon: float >= 0. If `None`, defaults to `backend.epsilon()`.
        decay: float >= 0. Learning rate decay over each update.

    # References
        - [Adaptive Subgradient Methods for Online Learning and Stochastic
        Optimization](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
    rT   NrU   c                     t                      j        di | t          j        | j        j                  5  t          j        |d          | _        t          j        |d          | _        t          j        ddd          | _	        d d d            n# 1 swxY w Y   |t          j
                    }|| _
        || _        d S 	Nr\   r]   r_   r   rW   rX   rY   r$   )r`   r   r   ra   rb   rL   rc   r\   r_   rX   r   rd   )r   r\   r   r_   r   rb   s        r   r   zAdagrad.__init__R  s   ""6""" 788 	 	&r555DG )%g>>>DJ%.|  DO	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ?o''G"   ABBBc                 D    d |D             }d |D             }|| _         |S )Nc                 6    g | ]}t          j        |          S r$   ri   rk   s     r   r,   z/Adagrad._create_all_weights.<locals>.<listcomp>`  rl   r   c                 6    g | ]}t          j        |          S r$   rn   rp   s     r   r,   z/Adagrad._create_all_weights.<locals>.<listcomp>a  "    AAAe,,AAAr   r   )r   r   rr   r   s       r   r   zAdagrad._create_all_weights_  s5    77777AA&AAA#r   c           
      V   |                      ||          }|                     |          }t          j        j                            | j        d          g| _        | j        }| j	        dk    rB|dd| j
        t          j        | j        t          j        | j
                            z  z   z  z  }t          |||          D ]\  }}}|t          j        |          z   }	| j                            t          j        j                            ||	                     |||z  t          j        |	          | j        z   z  z
  }
t)          |dd           |                    |
          }
| j                            t          j        j                            ||
                     | j        S ru   )r3   r   r)   ry   rz   r{   rX   r   r\   rd   r_   r|   r   rZ   r8   r   r:   r}   r   r   r~   rx   r   s              r   r    zAdagrad.get_updatese  s   ""400//77	//CCDW!!jgdow}TZ/H/HIIJJB 65,77 		? 		?GAq!	!$EL	 3 3Au = =>>>Q',u"5"5"DEEE q,--9U++L	 3 3Au = =>>>>|r   c                    t          t          j        | j                            t          t          j        | j                            | j        d}t                                                      }t          t          |
                                          t          |
                                          z             S )N)r\   r_   r   )r   r   r   r\   r_   r   r`   rH   r   r   r   r   s      r   rH   zAdagrad.get_config  s    )$'22337,TZ8899|
 

 gg((**D**,,--V\\^^0D0DDEEEr   )rT   NrU   r   r   s   @r   r   r   =  s         (# # # # # #    :F F F F F F F F Fr   r   c                   :     e Zd ZdZd
 fd	Zd Zd Z fd	Z xZS )AdadeltaaZ  Adadelta optimizer.

    Adadelta is a more robust extension of Adagrad
    that adapts learning rates based on a moving window of gradient updates,
    instead of accumulating all past gradients. This way, Adadelta continues
    learning even when many updates have been done. Compared to Adagrad, in the
    original version of Adadelta you don't have to set an initial learning
    rate. In this version, initial learning rate and decay factor can
    be set, as in most other Keras optimizers.

    It is recommended to leave the parameters of this optimizer
    at their default values.

    Arguments:
      lr: float >= 0. Initial learning rate, defaults to 1.
          It is recommended to leave it at the default value.
      rho: float >= 0. Adadelta decay factor, corresponding to fraction of
          gradient to keep at each time step.
      epsilon: float >= 0. Fuzz factor.
        If `None`, defaults to `backend.epsilon()`.
      decay: float >= 0. Initial learning rate decay.

    References:
        - [Adadelta - an adaptive learning rate
        method](http://arxiv.org/abs/1212.5701)
    rw   ffffff?NrU   c                     t                      j        di | t          j        | j        j                  5  t          j        |d          | _        t          j        |d          | _        t          j        ddd          | _	        d d d            n# 1 swxY w Y   |t          j
                    }|| _        || _
        || _        d S r   )r`   r   r   ra   rb   rL   rc   r\   r_   rX   r   r   rd   r   s         r   r   zAdadelta.__init__  s	   ""6""" 788 	 	&r555DG )%g>>>DJ%.|  DO	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ?o''G"r   c                 f    d |D             }d |D             }d |D             }||z   | _         ||fS )Nc                 6    g | ]}t          j        |          S r$   ri   rk   s     r   r,   z0Adadelta._create_all_weights.<locals>.<listcomp>  rl   r   c                 6    g | ]}t          j        |          S r$   rn   rp   s     r   r,   z0Adadelta._create_all_weights.<locals>.<listcomp>  r   r   c                 6    g | ]}t          j        |          S r$   rn   rp   s     r   r,   z0Adadelta._create_all_weights.<locals>.<listcomp>  s"    GGGugmE22GGGr   r   )r   r   rr   r   delta_accumulatorss        r   r   zAdadelta._create_all_weights  sU    77777AA&AAAGGGGG#&88///r   c           
         |                      ||          }t          j        j                            | j        d          g| _        |                     |          \  }}| j        }| j	        dk    rB|dd| j
        t          j        | j        t          j        | j
                            z  z   z  z  }t          ||||          D ]}\  }}}	}
| j        |	z  d| j        z
  t          j        |          z  z   }| j                            t          j        j                            |	|                     |t          j        |
| j        z             z  t          j        || j        z             z  }|||z  z
  }t+          |dd           |                    |          }| j                            t          j        j                            ||                     | j        |
z  d| j        z
  t          j        |          z  z   }| j                            t          j        j                            |
|                     | j        S ru   )r3   r)   ry   rz   r{   rX   r   r   r\   rd   r_   r|   r   rZ   r8   r   r   r:   r}   r   r   r~   rx   )r   r   r   r2   r   r   r\   r?   r&   r   d_ar   r   r   new_d_as                  r   r    zAdadelta.get_updates  s   ""400	//CCD+/+C+CF+K+K((W!!jgdow}TZ/H/HIIJJB  E<);
 
 	C 	CLAq!S HqLC$(Nbill#BBEL	 3 3Au = =>>> ,sT\1223,ut|3445 
 VOE q,--9U++L	 3 3Au = =>>> hnDH	&8I8I'IIGL	 3 3C A ABBBB|r   c                    t          t          j        | j                            | j        t          t          j        | j                            | j        d}t                                                      }t          t          |                                          t          |                                          z             S r   r   r   s      r   rH   zAdadelta.get_config  s    )$'223387,TZ8899|	
 
 gg((**D**,,--V\\^^0D0DDEEEr   )rw   r   NrU   r   r   s   @r   r   r     s         6# # # # # #0 0 0( ( (TF F F F F F F F Fr   r   c                   F     e Zd ZdZ	 	 	 	 	 	 d fd	Zd	 Zd
 Z fdZ xZS )Adama%  Adam optimizer.

    Default parameters follow those provided in the original paper.

    Args:
      lr: float >= 0. Learning rate.
      beta_1: float, 0 < beta < 1. Generally close to 1.
      beta_2: float, 0 < beta < 1. Generally close to 1.
      epsilon: float >= 0. Fuzz factor.
        If `None`, defaults to `backend.epsilon()`.
      decay: float >= 0. Learning rate decay over each update.
      amsgrad: boolean. Whether to apply the AMSGrad variant of this algorithm
        from the paper "On the Convergence of Adam and Beyond".
    r   r   +?NrU   Fc                 
    t                      j        d
i | t          j        | j        j                  5  t          j        ddd          | _        t          j        |d          | _        t          j        |d          | _	        t          j        |d          | _
        t          j        |d	          | _        d d d            n# 1 swxY w Y   |t          j                    }|| _        || _        || _        d S Nr   rW   rX   rY   r\   r]   beta_1beta_2r_   r$   )r`   r   r   ra   rb   rL   rc   rX   r\   r   r   r_   r   rd   amsgrad)	r   r\   r   r   r   r_   r   r   rb   s	           r   r   zAdam.__init__  s9    	""6""" 788 	? 	?%.|  DO &r555DG!*6AAADK!*6AAADK )%g>>>DJ	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? ?o''G"   B	CCCc                     d |D             }d |D             }| j         rd |D             }nd |D             }| j        g|z   |z   |z   | _        |||fS )Nc                     g | ]<}t          j        t          j        |          t          j        |                     =S r   r   rk   s     r   r,   z,Adam._create_all_weights.<locals>.<listcomp>  L     
 
 
 M'+A..gmA6F6FGGG
 
 
r   c                     g | ]<}t          j        t          j        |          t          j        |                     =S r   r   rk   s     r   r,   z,Adam._create_all_weights.<locals>.<listcomp>   r   r   c                     g | ]<}t          j        t          j        |          t          j        |                     =S r   r   rk   s     r   r,   z,Adam._create_all_weights.<locals>.<listcomp>%  sL        g/22'-:J:JKKK  r   c                 6    g | ]}t          j        d           S )rv   rn   )r%   _s     r   r,   z,Adam._create_all_weights.<locals>.<listcomp>*  s"    666!W]1%%666r   )r   rX   r   )r   r   msvsvhatss        r   r   zAdam._create_all_weights  s    
 

 
 

 

 
 
 < 	7   EE
 76v666E(2-2U:2u}r   c           
         |                      ||          }g | _        | j        }| j        dk    rB|dd| j        t          j        | j        t          j	        | j                            z  z   z  z  }t          j
        t
          j        j                            | j        d          g          5  t          j        | j        t          j                              }d d d            n# 1 swxY w Y   |t          j        dt          j        | j        |          z
            dt          j        | j        |          z
  z  z  }|                     |          \  }}}	t)          |||||	          D ]\  }
}}}}| j        |z  d| j        z
  |z  z   }| j        |z  d| j        z
  t          j        |          z  z   }| j        rxt          j        ||          }|
||z  t          j        |          | j        z   z  z
  }| j                            t
          j        j                            ||                     n%|
||z  t          j        |          | j        z   z  z
  }| j                            t
          j        j                            ||                     | j                            t
          j        j                            ||                     |}t7          |
dd           |
                    |          }| j                            t
          j        j                            |
|                     | j        S Nr   rw   rv   rx   )r3   r   r\   rd   r_   r)   r|   rX   r   rZ   control_dependenciesry   rz   r{   floatxr   powr   r   r   r8   r   r   maximumr   r:   r}   r~   rx   )r   r   r   r2   r\   tlr_tr   r   r   r?   r&   r   r   vhatm_tv_tvhat_tp_tr   s                       r   r    zAdam.get_updates.  s)   ""400W!!jgdow}TZ/H/HIIJJB $Y\$$T_a889
 
 	; 	; )9)9::A	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; Lrvdk155566RVDK+++-

 0088B #FE2r5 A A 	? 	?Aq!Q;?sT['8A&==C;?sT['8BIaLL&HHC| JD#..$*V(<(<t|(KLL##BIL$7$7f$E$EFFFF$*S(9(9DL(HIIL	 3 3As ; ;<<<L	 3 3As ; ;<<<E q,--9U++L	 3 3Au = =>>>>|   /,C''C+.C+c                 *   t          t          j        | j                            t          t          j        | j                            t          t          j        | j                            t          t          j        | j                            | j        | j        d}t                      
                                }t          t          |                                          t          |                                          z             S )N)r\   r   r   r_   r   r   )r   r   r   r\   r   r   r_   r   r   r`   rH   r   r   r   r   s      r   rH   zAdam.get_config\  s    )$'2233G-dk::;;G-dk::;;7,TZ8899||
 
 gg((**D**,,--V\\^^0D0DDEEEr   )r   r   r   NrU   Fr   r   s   @r   r   r     s         "      2  &, , ,\
F 
F 
F 
F 
F 
F 
F 
F 
Fr   r   c                   D     e Zd ZdZ	 	 	 	 	 d fd	Zd Zd	 Z fd
Z xZS )Adamaxa  Adamax optimizer from Adam paper's Section 7.

    It is a variant of Adam based on the infinity norm.
    Default parameters follow those provided in the paper.

    Args:
      lr: float >= 0. Learning rate.
      beta_1/beta_2: floats, 0 < beta < 1. Generally close to 1.
      epsilon: float >= 0. Fuzz factor.
        If `None`, defaults to `backend.epsilon()`.
      decay: float >= 0. Learning rate decay over each update.
    Mb`?r   r   NrU   c                     t                      j        d
i | t          j        | j        j                  5  t          j        ddd          | _        t          j        |d          | _        t          j        |d          | _	        t          j        |d          | _
        t          j        |d	          | _        d d d            n# 1 swxY w Y   |t          j                    }|| _        || _        d S r   )r`   r   r   ra   rb   rL   rc   rX   r\   r   r   r_   r   rd   )r   r\   r   r   r   r_   r   rb   s          r   r   zAdamax.__init__w  s4    	""6""" 788 	? 	?%.|  DO &r555DG!*6AAADK!*6AAADK )%g>>>DJ	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? ?o''G"r   c                 x    d |D             }d |D             }d |D             }| j         g|z   |z   | _        ||fS )Nc                 6    g | ]}t          j        |          S r$   ri   rk   s     r   r,   z.Adamax._create_all_weights.<locals>.<listcomp>  rl   r   c                 6    g | ]}t          j        |          S r$   rn   rp   s     r   r,   z.Adamax._create_all_weights.<locals>.<listcomp>  "    777ugmE""777r   c                 6    g | ]}t          j        |          S r$   rn   rp   s     r   r,   z.Adamax._create_all_weights.<locals>.<listcomp>  r   r   rq   )r   r   rr   r   uss        r   r   zAdamax._create_all_weights  sZ    777777777777777(2-22vr   c           
         |                      ||          }g | _        | j        }| j        dk    rB|dd| j        t          j        | j        t          j	        | j                            z  z   z  z  }t          j
        t
          j        j                            | j        d          g          5  t          j        | j        t          j                              }d d d            n# 1 swxY w Y   |dt          j        | j        |          z
  z  }|                     |          \  }}t%          ||||          D ]A\  }	}
}}| j        |z  d| j        z
  |
z  z   }t          j        | j        |z  t          j        |
                    }|	||z  || j        z   z  z
  }| j                            t
          j        j                            ||                     | j                            t
          j        j                            ||                     |}t3          |	dd           |	                    |          }| j                            t
          j        j                            |	|                     C| j        S r   )r3   r   r\   rd   r_   r)   r|   rX   r   rZ   r   ry   rz   r{   r   r   r   r   r8   r   r   absr   r:   r}   r~   rx   )r   r   r   r2   r\   r   r   r   r   r?   r&   r   ur   u_tr   r   s                    r   r    zAdamax.get_updates  s|   ""400W!!jgdow}TZ/H/HIIJJB $Y\$$T_a889
 
 	; 	; )9)9::A	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; S26$+q1112))&11BfeR44 	? 	?JAq!Q;?sT['8A&==C*T[1_bfQii88CdSjC$,$677CL	 3 3As ; ;<<<L	 3 3As ; ;<<<E q,--9U++L	 3 3Au = =>>>>|r   c                    t          t          j        | j                            t          t          j        | j                            t          t          j        | j                            t          t          j        | j                            | j        d}t                      	                                }t          t          |                                          t          |                                          z             S )N)r\   r   r   r_   r   )r   r   r   r\   r   r   r_   r   r`   rH   r   r   r   r   s      r   rH   zAdamax.get_config  s    )$'2233G-dk::;;G-dk::;;7,TZ8899|
 
 gg((**D**,,--V\\^^0D0DDEEEr   )r   r   r   NrU   r   r   s   @r   r   r   i  s          # # # # # #.  & & &P	F 	F 	F 	F 	F 	F 	F 	F 	Fr   r   c                   D     e Zd ZdZ	 	 	 	 	 d fd	Zd Zd	 Z fd
Z xZS )Nadama  Nesterov Adam optimizer.

    Much like Adam is essentially RMSprop with momentum,
    Nadam is Adam RMSprop with Nesterov momentum.

    Default parameters follow those provided in the paper.
    It is recommended to leave the parameters of this optimizer
    at their default values.

    Args:
      lr: float >= 0. Learning rate.
      beta_1/beta_2: floats, 0 < beta < 1. Generally close to 1.
      epsilon: float >= 0. Fuzz factor.
        If `None`, defaults to `backend.epsilon()`.
    r   r   r   NMbp?c                     t                      j        di | t          j        | j        j                  5  t          j        ddd          | _        t          j        dd          | _        t          j        |d          | _	        t          j        |d	          | _
        t          j        |d
          | _        d d d            n# 1 swxY w Y   |t          j                    }|| _        || _        d S )Nr   rW   rX   rY   rw   
m_scheduler]   r\   r   r   r$   )r`   r   r   ra   rb   rL   rc   rX   r   r\   r   r   r   schedule_decay)r   r\   r   r   r   r   r   rb   s          r   r   zNadam.__init__  sE    	""6""" 788 	B 	B%.|  DO &.sFFFDO&r555DG!*6AAADK!*6AAADK	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B ?o''G,r   c                     d |D             }d |D             }d |D             }| j         | j        g|z   |z   | _        ||fS )Nc                 6    g | ]}t          j        |          S r$   ri   rk   s     r   r,   z-Nadam._create_all_weights.<locals>.<listcomp>  rl   r   c                 6    g | ]}t          j        |          S r$   rn   rp   s     r   r,   z-Nadam._create_all_weights.<locals>.<listcomp>  r   r   c                 6    g | ]}t          j        |          S r$   rn   rp   s     r   r,   z-Nadam._create_all_weights.<locals>.<listcomp>  r   r   )rX   r   r   )r   r   rr   r   r   s        r   r   zNadam._create_all_weights  s^    7777777777777779B>C2vr   c                 .   |                      ||          }g | _        t          j        t          j        j                            | j        d          g          5  t          j        | j        t          j
                              }d d d            n# 1 swxY w Y   | j        ddt          j        t          j        d          || j        z            z  z
  z  }| j        ddt          j        t          j        d          |dz   | j        z            z  z
  z  }| j        |z  }| j        |z  |z  }| j                            | j        |f           |                     |          \  }	}
t%          |||	|
          D ]\  }}}}|d|z
  z  }| j        |z  d| j        z
  |z  z   }|d|z
  z  }| j        |z  d| j        z
  t          j        |          z  z   }|dt          j        | j        |          z
  z  }d|z
  |z  ||z  z   }| j                            t          j        j                            ||                     | j                            t          j        j                            ||                     || j        |z  t          j        |          | j        z   z  z
  }|}t3          |dd           |                    |          }| j                            t          j        j                            ||                     | j        S )Nrv   rw   g      ?gQ?rx   )r3   r   r)   r   ry   rz   r{   rX   r|   r   r   r   r   cast_to_floatxr   r   r:   r   r8   r   r   r}   r\   r   r   r~   rx   )r   r   r   r2   r   momentum_cache_tmomentum_cache_t_1m_schedule_newm_schedule_nextr   r   r?   r&   r   r   g_primer   	m_t_primer   	v_t_primem_t_barr   r   s                          r   r    zNadam.get_updates  sJ   ""400$Y\$$T_a889
 
 	; 	; )9)9::A	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	;  ;vg,T22A8K4KLLNN

 "[*4001q5D<O2O 
 +;;O..1CC 	 	T_n=>>>))&11BfeR44 	? 	?JAq!Q3/0G+/S4;%6!$;;Cs_45I+/S4;%6")A,,$FFCsRVDK%;%;;<I&&,y89G L	 3 3As ; ;<<<L	 3 3As ; ;<<<dg'Y''$,6 C E q,--9U++L	 3 3Au = =>>>>|s   ,BBBc                    t          t          j        | j                            t          t          j        | j                            t          t          j        | j                            | j        | j        d}t                      	                                }t          t          |                                          t          |                                          z             S )N)r\   r   r   r   r   )r   r   r   r\   r   r   r   r   r`   rH   r   r   r   r   s      r   rH   zNadam.get_config6  s    )$'2233G-dk::;;G-dk::;;|"1
 
 gg((**D**,,--V\\^^0D0DDEEEr   )r   r   r   Nr   r   r   s   @r   r   r     s         $ - - - - - -.  8 8 8t	F 	F 	F 	F 	F 	F 	F 	F 	Fr   r   c                   \    e Zd ZdZddZd ZddZd Zd Zd Z	e
d	             Zd
 Zd ZdS )TFOptimizerz/Wrapper class for native TensorFlow optimizers.Nc                 2   || _         |                     |d           |St          j        | j        j                  5  t          j        ddd          | _        d d d            n# 1 swxY w Y   n|| _        |                     | j        d           d S )N	optimizerr]   r   rW   rX   rY   global_step)r  _track_trackabler   ra   rb   rL   rc   rX   )r   r  rX   s      r   r   zTFOptimizer.__init__E  s    "ik:::#DN$;<<  ")"2W<# # #              
 )DOdoMBBBBBs   A((A,/A,c                     |S )zBClip gradients according to the clipnorm and clipvalue attributes.r$   )r   r2   s     r   _clip_gradientszTFOptimizer._clip_gradientsQ  s	     r   c                 ,   t          |          s|t          d          ||nt          j                    }t          |          ra|5  t          |          s|                    |            |            }t          |          r
 |            }ddd           n# 1 swxY w Y   t          j                            |          }|rK|                    |||          }t          t          ||                    }| 
                    |           dS dS )z&Mimics the `OptimizerV2.minimize` API.Nz2`tape` is required when a `Tensor` loss is passed.)callabler   r)   GradientTapewatchnestflattengradientr   r8   apply_gradients)r   r   var_list	grad_losstaper2   grads_and_varss          r   minimizezTFOptimizer.minimizeV  s_   ~~ 	$,D   'ttR_->->D>> 	* * *)) )JJx(((tvvH%% *'xzzH* * * * * * * * * * * * * * * 7??8,, 	1MM$)<<E!#eX"6"677N  00000	1 	1s   	ABB!$B!c                 H    | j                             || j                   d S )Nr  )r  r  rX   )r   r  s     r   r  zTFOptimizer.apply_gradientsl  s2    && 	' 	
 	
 	
 	
 	
r   c                 8    | j                             ||          S r   )r  compute_gradientsr   s      r   	get_gradszTFOptimizer.get_gradsq  s    ~//f===r   c                 ~   t           j                                        rg | _        |s| j                            |          }n| j                            ||          }t           j        j        j        	                                }| j        
                    ||          }n||s7t           j        j                            | j        d          g| _        | j        S g | _        | j                            ||          }| j        
                    || j                  }| j                            |           | j        S )Nrv   r  )r)   
distributehas_strategyr   r  r  ry   rz   trainget_global_stepr  r{   rX   r:   )r   r   r   r2   r  
opt_updates         r   r    zTFOptimizer.get_updatest  s   =%%'' 	DL G 88>>88vFF),,<<>>K77{KKJJ $ "	 7 7 K KL|# DLN44T6BBE774? 8  J 	J'''|r   c                     t           r   r   rC   s    r   r   zTFOptimizer.weights  s    !!r   c                     t           r   r   rC   s    r   rH   zTFOptimizer.get_config  r!   r   c                     t           r   r   rF   s     r   rK   zTFOptimizer.from_config  r!   r   r   )NN)rL   rM   rN   rO   r   r	  r  r  r  r    propertyr   rH   rK   r$   r   r   r  r  B  s        99
C 
C 
C 
C  
1 1 1 1,
 
 

> > >  : " " X"" " "" " " " "r   r  )rO   tensorflow.compat.v2ry   v2r)   kerasr   r   rS   r   r   r   r   r   r   __internal__tracking	Trackabler  sgdrmspropadagradadadeltaadamadamaxnadamr$   r   r   <module>r3     sr  " 
 " ! ! ! ! ! ! ! !      E E E E E E E EPJF JF JF JF JF) JF JF JFZJF JF JF JF JFi JF JF JFZLF LF LF LF LFi LF LF LF^cF cF cF cF cFy cF cF cFLtF tF tF tF tF9 tF tF tFn`F `F `F `F `FY `F `F `FFsF sF sF sF sFI sF sF sFlW" W" W" W" W")R_5? W" W" W"x 


	r   