Vhe1dZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZddlmZddlZd d lmZd d lmZmZd d lmZmZd dejeejfdefdZGdde Z Gdde Z!Gdde Z"GddZ#d d d ddZ$GddZ%dZ&Gdde Z'dS)!z Support for PyTorch lightning. You should just replace the call to `Trainer(...)` with `get_trainer(...)`. For using `dora.log.LogProgress` as a progress bar with PL, see `PLLogProgress`. N)LightningModule)Callback)ProgressBarBase)ClusterEnvironment)Trainer)from_argparse_args)distrib)get_xpis_xp)bold LogProgressTmetricsepochci}|D]\}}|r|dr|s|dr5|ds|dr|ddd}t|tjr,|dkr|}|||<|S)aFilters metrics before formatting, in particular to remove the `_step` or `_epoch` suffix. This will also convert torch tensors to float. Args: metrics: dict given by PL. epoch: if True, keep only epoch level metrics, otherwise, keep only step level metrics. _step_epoch_r r)itemsendswithrsplit isinstancetorchTensornumelitem)rroutkeyvalues T/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/dora/lightning.py_filter_metricsr!s Cmmoo   U  S\\'**   h//   <<  (CLL$:$: (**S!$$Q'C eU\ * * !u{{}}/A/AJJLLEC JceZdZfdZdefdZedefdZdefdZ deddfdZ defd Z d eddfd Z defd Z defd ZedefdZedefdZedefdZxZS)DoraEnvironmentcttj|_tjdSN)super__init__r get_distrib_specspecset_distrib_envself __class__s r r(zDoraEnvironment.__init__4s> ,.. !!!!!r"returncdSNTr-s r creates_childrenz DoraEnvironment.creates_children9str"cdSr1r2r3s r creates_processes_externallyz,DoraEnvironment.creates_processes_externally<str"c|jjSr&)r* world_sizer3s r r8zDoraEnvironment.world_size@ y##r"sizeNcdSr&r2)r-r:s r set_world_sizezDoraEnvironment.set_world_sizeC r"c|jjSr&)r*rankr3s r global_rankzDoraEnvironment.global_rankFs y~r"r?cdSr&r2)r-r?s r set_global_rankzDoraEnvironment.set_global_rankIr=r"c|jjSr&)r* local_rankr3s r rDzDoraEnvironment.local_rankLr9r"c|jjSr&)r* node_rankr3s r rFzDoraEnvironment.node_rankOs y""r"cdSNFr2r2r"r detectzDoraEnvironment.detectRsur"c&tjdS)N MAIN_ADDR)osenvironr3s r main_addresszDoraEnvironment.main_addressVsz+&&r"c@ttjdS)N MAIN_PORT)intrLrMr3s r main_portzDoraEnvironment.main_portZs2:k*+++r")__name__ __module__ __qualname__r(boolr4propertyr6rQr8r<r@rBrDrF staticmethodrIstrrNrR __classcell__r.s@r r$r$3s""""" $dX$C$$$$ 3 4    S C D    $C$$$$#3####D\'c'''X',3,,,X,,,,,r"r$c$eZdZdZdZdZdZdS)DoraCheckpointSyncz>Make sure Dora history, and checkpoint state are in sync. c,t|_dSr&)r xpr3s r r(zDoraCheckpointSync.__init__bs((r"cT|d}|jj|dS)Ndora_link_history)r_linkupdate_history)r-trainer pl_module checkpointhistorys r on_load_checkpointz%DoraCheckpointSync.on_load_checkpointes+01  ##G,,,,,r"cj|jjj|d<|jj|d<|jj|d<|S)Nradora_sigdora_cfg)r_rbrgsigcfgr-rdrerfs r on_save_checkpointz%DoraCheckpointSync.on_save_checkpointis6*.',*> &'!% :!% :r"N)rSrTrU__doc__r(rhror2r"r r]r]_sK---r"r]c4eZdZdZfdZdZdZdZxZS)DoraHistoryLoggerz,Save metrics to Dora using the XP link. cxttj|_dSr&)r'r(r rbr,s r r(zDoraHistoryLogger.__init__ss* HHM r"cd|_dSr1 _first_validr-rdres r on_fit_startzDoraHistoryLogger.on_fit_startws r"cd|_dSrHrurws r on_train_epoch_startz&DoraHistoryLogger.on_train_epoch_startzs!r"c||jrdS|j}t|d}|j|dS)NTr)rvlogged_metricsr!rb push_metrics)r-rdrers r on_epoch_endzDoraHistoryLogger.on_epoch_end}sH    F(!'666 w'''''r") rSrTrUrpr(rxrzrrZr[s@r rrrrpso"""""!!!"""(((((((r"rrceZdZdZdS)_DummySLURMConnectorcdSr&r2r3s r register_slurm_signal_handlersz3_DummySLURMConnector.register_slurm_signal_handlersr=r"N)rSrTrUrr2r"r rrs#     r"r) auto_resumeadd_dora_loggerno_unfinished_epochsc~tstdtj}t |dr|j}t |dt j|dgt|zfi|}|d=| dpg}t}ttj |}|dkr||dgz }||d<| ddpg} | t#| |d<|d td |d dkrtd ||d <|jj|d <t)j|d <|r'|dt-|d} |rC| At)jdz } | rt3| } nd} | |d<tdi|} |rt5| _| S)aReturn a PL trainer, adding the necessary glue code to make everything works. The arguments are exactly the same as for `pytorch_lightning.trainer.Trainer`, with a few extras documented after. ..note:: You should not pass `gpus=` or `num_nodes=` arguments as those will be filled by Dora. Args: auto_resume (bool): if True, automatically resume previous checkpoints. You are still responsible for creating the `ModelCheckpoint` callback, this only handles the `resume_from_checkpoint` part. add_dora_logger (bool): if True, adds a Dora Logger to automatically forward the metrics (those logged with per_epoch=True), otherwise pushing metrics will be up to you. no_unfinished_epochs (bool): if True, deactivates SLURM signal handling by PL, which can result in half finished epoch with each interruption. It is recommended to instead dump a checkpoint every epoch and resume from that one so that training is reliable. z.This can only be called from inside a Dora XP. __wrapped__Nr-pluginsr ddp callbacksgpuszCYou cannot specify the number of GPUs, as this is provided by Dora. num_nodeszDYou cannot specify the number of nodes, as this is provided by Dora.default_root_dirresume_from_checkpointz last.ckptr2)r RuntimeErrorrr(hasattrrinspect getcallargslistpopr$minrcuda device_countr8appendr]r*rr folderrrgetis_filerYrslurm_connector)rrrargskwargsinitrenvrrrlastresumerds r get_trainerrsM* 77MKLLL  D $ & &  $ & &  vT ': E Ef E EFvjj##)rG   C uz&&((#..*:*: ; ;D ~~!C<F9 ;--3I '))***#F; f~!`aaa kaabbbF6N(,F;!'F 8{""#4#6#6777#ZZ(@AA2-5xx, <<>> YYFFF+1'(G9"6"8"8 Nr"cJeZdZejejdZdS) _Interceptc"||_||_dSr&)rr)r-rrs r r(z_Intercept.__init__s  r"N)rSrTrU functoolswrapsrr(r2r"r rrs=Y_W%&&'&r"rc Ttt|fi|}t|ji|jSr&)rrrrr)rr intercepts r trainer_from_argparse_argsrs1":t>>v>>I   ;)*: ; ;;r"c eZdZdZdfd Zdeddffd ZfdZede fdZ dd e j ee j fded efd Zed ZdZfdZfdZdZfdZfdZdZdZfdZfdZfdZdZdZdZxZS) PLLogProgressz<`dora.log.LogProgress` support for Pytorch-Lightning. r/Nc rt||_||_d|_dSr&)r'r(loggerr _pl_module)r-rrr.s r r(zPLLogProgress.__init__s2   8<r"stagecjt|||||_g|_dSr&)r'setupr_replay_history)r-rdrerr.s r rzPLLogProgress.setups2  gy%000#02r"cht||d|_d|_dS)NFT)r'rx _in_trainrvr-rdrer.s r rxzPLLogProgress.on_fit_starts2 Wi000 r"c"|jJ|jSr&)rr3s r rezPLLogProgress.pl_modules***r"Frrci}|D]-\}}t|trt|d||<.|S)agDefault method to format metrics for displaying in the progress bar. To customize, you can define a `format_metrics()` method on your Lightning module. Args: metrics: dict of metrics given by PL. stage: "train" or "valid". epoch: if True, provided metrics are for the end of epoch summary. z.5f)rrfloatformat)r-rrrrrrs r format_metricszPLLogProgress.format_metricssO!--// 0 0JC%'' 0!%//C r"c8t|jd|jS)Nr)getattrrerr3s r _format_metricszPLLogProgress._format_metrics st~'79LMMMr"c|jd|j|dkrdnd|d|jjdzz}|dkrt |j}n-|dkrt |j}ntd|t|}t|j|f||d |j |_ t|j dS) NzF----------------------------------------------------------------------trainz Training...z Validating...z | Epoch r validzInvalid stage )totalname)rinfo capitalizerd current_epochrQtotal_train_batchestotal_val_batchesrrangerrlogprogiter)r-rrrloaders r _on_epoch_startzPLLogProgress._on_epoch_starts """ %7*:*:PPP!!$P 0JQ0N$P$PP G  011EE g  .//EE77788 8u"4;^e$^^RVR]^^  T\r"c|dd|_d|_t||S)NrTF)rrrvr'rzrs r rzz"PLLogProgress.on_train_epoch_starts@ W%%%!ww++GY???r"cr|dt||SNr)rr'on_validation_epoch_startrs r rz'PLLogProgress.on_validation_epoch_start$s1 W%%%ww00)DDDr"c||j|j}t|d}|||d}|jjdi|t|jdS)NFr|r2) get_metricsrdrer!rrupdatenext)r-rr formatteds r _on_batch_endzPLLogProgress._on_batch_end(sx""4<@@!'777((%u(EE  ((i((( T\r"cdtj|i||ddS)Nr)r'on_train_batch_endrr-rrr.s r rz PLLogProgress.on_train_batch_end/s9""D3F333 7#####r"cdtj|i||ddSr)r'on_validation_batch_endrrs r rz%PLLogProgress.on_validation_batch_end3s9''8888 7#####r"c>|dkr0|jjjjdd}n4|jjjjjdd}t |d}|||jj|dS)NrFlogr|) rdfit_loop epoch_loop_resultsrval_loopr!_show_epoch_summaryr)r-rrs r _on_stage_endzPLLogProgress._on_stage_end7s G  l+6?GGNNuUGGl+6?HPPQVWWX]^G!'777    (BGLLLLLr"c ^|j|||f|||d}|}dd|D}|jt|d|dzd|dS)NTr|z | c3NK|] \}}|d|V!dS)=N)r).0rvals r z4PLLogProgress._show_epoch_summary..DsO  ,4Cs~~ ' '# ' '      r"z Summary | End of Epoch r ) rrrrjoinrrrr )r-rrrrrsummarys r rz!PLLogProgress._show_epoch_summary@s ##UE7$;<<<((%t(DD !!**  8A8I8I      VVuqyVVWVVWWXXXXXr"ct|||js |jsJ|js|dd|_dSdSNrF)r' on_train_endrrvrrs r on_validation_startz!PLLogProgress.on_validation_startIsg Wi000~2!2222  #   w ' ' '"DNNN # #r"ct|||jr|dd|_dSr)r'rrrrs r rzPLLogProgress.on_epoch_endPsF Wi000 > (   w ' ' 'r"cvt|||ddSr)r'on_validation_endrrs r rzPLLogProgress.on_validation_endVs7 !!'9555 7#####r"cdSr&r2r3s r disablezPLLogProgress.disableZs  r"c|dg}|r|jd|D] }|j| dS)Ndora_replay_historyzReplaying past metrics...)rrrr)r-rdrerfreplay_historysteps r rhz PLLogProgress.on_load_checkpoint_sa#(=rBB  : K  8 9 9 9" , ,D $D $d + + + , ,r"c|j|d<|S)Nr)rrns r roz PLLogProgress.on_save_checkpointfs,0,@ ()r")r/N)F) rSrTrUrpr(rYrrxrWrretpDictAnyrVrrrrzrrrrrrrrrrrhrorZr[s@r rrs; ====== 3s3t333333 !!!!! ?X 27bgc26k&:!*."NNXN   @@@@@ EEEEE$$$$$$$$$$MMMYYY##### $$$$$   ,,,r"r)T)(rprrrLtypingrpytorch_lightningrpytorch_lightning.callbacksr$pytorch_lightning.callbacks.progressr&pytorch_lightning.plugins.environmentsrpytorch_lightning.trainerr$pytorch_lightning.utilities.argparserrr r_r r rr rrrYrrVr!r$r]rrrrrrrr2r"r rsl  ------000000@@@@@@EEEEEE------CCCCCC """"""""RWS"&[1$*),),),),),(),),),X"((((((((,        $(TXCCCCCL<<< IIIIIOIIIIIr"