$h3!vddlZddlmZddlmZmZmZmZmZmZm Z m Z m Z ddl m Z ddlmZmZddgZd"d ZGd deZd ed e d ed e d e_ d#deedeedeedeedeedeedeedededededededededef dZdeedeedeedeedeedeedededededededededefd Zdeedeedeedeedeedeedededededededededefd!ZdS)$N)Tensor) Optimizer_use_grad_for_differentiable _get_value_default_to_fused_or_foreach_differentiable_doc _foreach_doc _maximize_doc_capturable_doc _view_as_real) is_compiling)ListOptionalASGDasgdcft|tjstj||S|S)Ndevice) isinstancetorchrtensor)xrs P/var/www/html/auto_sub_bot/venv/lib/python3.11/site-packages/torch/optim/asgd.py _to_tensorr s0 a & &.|Af---- Hc xeZdZ ddeed ed ed effd Zfd ZdZeddZ xZ S)r{Gz?-C6??.ArNFforeachmaximizedifferentiable capturablec d|kstd|d|kstd||dur| rtdt|||||||| |  } t|| dS)NgzInvalid learning rate: zInvalid weight_decay value: F0Capturable not supported with single tensor ASGD) lrlambdalphat0 weight_decayr"r#r$r%) ValueErrordictsuper__init__) selfparamsr(r)r*r+r,r"r#r$r%defaults __class__s rr0z ASGD.__init__sbyy;r;;<< <l""JLJJKK K e   OPP P%)!     *****rct||jD]Z}|dd|dd|dd|dd[t |j}t|dkotj |dd}|s;|D]8}tj t|dtj |d<9t|dkotj |dd }|s.|D]+}tj |d tj |d <,t|dkotj |dd }|s;|D]:}tj t|d tj |d <9dSdS) Nr"r#Fr$r%rstep)dtypeetamu) r/ __setstate__ param_groups setdefaultliststatevalueslenr is_tensorrfloatfloat32) r1r>group state_valuesstep_is_tensors eta_is_tensor mu_is_tensorr4s rr:zASGD.__setstate__4s U###& 2 2E   Y - - -   Z / / /   -u 5 5 5   \5 1 1 1 1DJ--//00 l++q0 eo OF #7 7  P! P P!Lqy)9)9OOO& \**a/ U_ OE "6 6  G! G G <% FFF%L))Q. EO OD !5 5  L! L L,uQtW~~U]KKK$ L L L LrcVd}|dD]} | j|tj| z}|| | jjrt d|| j|j| } t| dkrtjd| j tj | d<tj |d| j tj | d <tj d| j tj | d <tj | tj | d <|| d || d || d || d|S) NFr2z&ASGD does not support sparse gradientsr)rr7r6r(r8r9) memory_formatax)gradr is_complexappend is_sparse RuntimeErrorr>r@zerosrrCrones zeros_likepreserve_format) r1rDparams_with_gradgradsmusaxsetas state_steps has_complexpr>s r _init_groupzASGD._init_groupOs| x 2 2Av!u/222  ''***6#Q&'OPPP QV$$$ 1 u::??$)K185=$Y$Y$YE&M#(<d AHTYTa#b#b#bE%L"'*R "V"V"VE$K"'"2)>###E$K 5;''' 5;''' E%L)))""5=111rcd}|5tj5|}dddn #1swxYwY|jD]~}g}g}g}g}g}g} |||||||| } t |||||| |d|d|d|d|d|d|d|d |d | |S) zPerform a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nr)r(r+r*r,r"r#r$r%) r)r(r+r*r,r"r#r$r%r])r enable_gradr;r_r) r1closurelossrDrWrXrYrZr[r\r]s rr6z ASGD.stepisQ  "$$ ! !wyy ! ! ! ! ! ! ! ! ! ! ! ! ! ! !&  E! ECCDK**52BE3PSUY[fggK  Gn;;Gn">2i(z*$%56 .'!     & s /33) rrr r!rNFFFN) __name__ __module__ __qualname__rboolr0r:r_rr6 __classcell__)r4s@rrrs  "&$  + +$ + + + + + + + + +DLLLLL64")))"!)))))rahImplements Averaged Stochastic Gradient Descent. It has been proposed in `Acceleration of stochastic approximation by averaging`_. Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-2) lambd (float, optional): decay term (default: 1e-4) alpha (float, optional): power for eta update (default: 0.75) t0 (float, optional): point at which to start averaging (default: 1e6) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) z z For ASGD, capturable is only supported when foreach is True. .. _Acceleration of stochastic approximation by averaging: https://dl.acm.org/citation.cfm?id=131098 Fr2rXrZrYr[r\r"r#r$r%r]r)r(r+r*r,c b|t||d\}}|r-tjrt d|r&tjst }n&| rt st dt}|||||||| | | ||||| | dS)znFunctional API that performs asgd algorithm computation. See :class:`~torch.optim.ASGD` for details. NF) use_fusedz6torch.jit.script not supported with foreach optimizersr') r)r(r+r*r,r#r$r%r])rrjit is_scriptingrR_multi_tensor_asgdr_single_tensor_asgd)r2rXrZrYr[r\r"r#r$r%r]r)r(r+r*r,_funcs rrrs21&.TYZZZ 7U59))++USTTT#uy--//#!  Slnn SQRR R"D     !%rc t|D]\}}||}| s|n| }||}||}||}||}tj|r> RWWYY!^^ GGEIIbMM%%b)) * * * * HHUOOORA T(9$9e#CDEE 'AAtby 1 1122 I$$rc t|dkrdS| r Jdtj||||||g}|D]\\}}\\}}}}}}}| rt j|}t |}|rt||||djr,t j |t j dddnt j |d| dkrK| rt j ||| |}nt j ||| }t j |||nt j |||}t j |||d ~t j ||}t j |||~| rt j ||}t j|dt j|t j||~t j|| }t j||t j||t j |dt j|t j||t j||0|d}g}g}t)t|D]q}t+|d||z|| zzzz |}||t+dt/d||z z |}||rt j||t j||dS) Nrz#_foreach ops don't support autogradg?cpurrsr)value)r@r"_group_tensors_by_device_and_dtypeitemsr _foreach_negr=r is_cpu _foreach_add_r _foreach_add_foreach_addcmul_ _foreach_sub_foreach_maximum__foreach_reciprocal__foreach_copy_ _foreach_pow _foreach_mul_ryrangerrPr})r2rXrZrYr[r\r)r(r+r*r,r#r$r%r]grouped_tensorsrrpgrouped_params grouped_grads grouped_axs grouped_mus grouped_etasgrouped_state_steps intermediatenew_musnew_etasr6r~rrs rrnrn s& 6{{aDDDDD BFESVX[]acnCoppO4C4I4I4K4KO7O7 1&!0, {K *Q  >!.}==M]++  F .- E E E q ! ( 8   3U\#e5T5T5T\_ ` ` ` ` `   3Q 7 7 7 1   e#M>VVVV, $1-Wcddd   nE J J J J J -m^SXYYYL  lRTUUUU ).+FF   \;GGG   7()rs jjjjjjjjjjjjjjjjjjjjjj%%%%%%!!!!!!!! 6     BBBBB9BBBJ    ! B# 66 L6 <6 f6 f 6 v, 6 f 6d^66666 6 !6" #6$ %6&'6666r6 L6 <6 f6 f 6 v, 6 f 6 6 6 6 66666 !6666rh7 Lh7 <h7 fh7 f h7 v, h7 f h7 h7 h7 h7 h7h7h7h7h7 !h7h7h7h7h7h7r