
    9i&                         d dl Z d dlZddlmZ ddlmZmZmZ ddlm	Z	 ddl
mZ  ede      r* e	       r#d dlmc mZ d d	lmZmZ d d
lmZ d dlmZ d dlmZ  ee      ZddZddZddZddZy)    N   )
get_logger   )FSDP_PYTORCH_VERSION
MODEL_NAMEOPTIMIZER_NAME)is_torch_distributed_available)is_torch_versionz>=)DefaultLoadPlannerDefaultSavePlanner)!load_sharded_optimizer_state_dict)FullyShardedDataParallel)StateDictTypec                    t        j                  |d       | j                  t        j                  k(  r1|j
                  dkD  }|| j                  _        || j                  _        t        j                  || j                  | j                  | j                        5  |j                         }| j                  t        j                  k(  r|dk(  r	t         dnt         d| d}t         j                  j                  ||      }|j                  dk(  rt         j#                  d|        t%        j&                  ||       t         j#                  d|        n|| j                  t        j(                  k(  r|dk(  rt         d	|j                   dnt         d| d	|j                   d}t         j                  j                  ||      }t         j#                  d|        t%        j&                  ||       t         j#                  d|        n| j                  t        j*                  k(  rt         j                  j                  |t         d|       }	t        j                  |	d       t         j#                  d|	        d
|i}t-        j.                  |t-        j0                  |	      t3                      t         j#                  d|	        d d d        y # 1 sw Y   y xY w)NTexist_okr   r   .bin_zSaving model to zModel saved to _rankmodel
state_dictstorage_writerplanner)osmakedirsstate_dict_typer   FULL_STATE_DICTnum_processesstate_dict_configoffload_to_cpu
rank0_onlyFSDPoptim_state_dict_configr   r   pathjoinprocess_indexloggerinfotorchsaveLOCAL_STATE_DICTSHARDED_STATE_DICTdist_cpsave_state_dictFileSystemWriterr   )
fsdp_pluginacceleratorr   
output_dirmodel_indexis_multi_processr   weights_nameoutput_model_fileckpt_dirs
             e/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/accelerate/utils/fsdp_utils.pysave_fsdp_modelr:   #   s}   KK
T*""m&C&CC '44q87G%%43C%%0			{**K,I,I;KnKn
  6 %%'
&&-*G*GG2=2Bj\.:,VWXcWddhHiL "Z F((A-./@.ABC

:'89o.?-@AB((M,J,JJ !# ,eK$=$=#>dC"|1[M{7P7P6QQUV 
 !#Z FKK*+<*=>?JJz#45KK/*;)<=>((M,L,LLww||J:,a}0MNHKK40KK*8*56!:.J##%&77A*,
 KK/(45A 6  6  6s   H<K!!K*c                 @   |j                          | j                  t        j                  k(  r1|j                  dkD  }|| j
                  _        || j
                  _        t        j                  || j                  | j
                  | j                        5  | j                  t        j                  k(  rt        |      t        k7  r0|j                  dk7  r!| j                  st        d      	 d d d        y |dk(  r	t         dnt         d| d}t        j                   j#                  ||      }t$        j'                  d|        t)        j*                  |      }t$        j'                  d|        n| j                  t        j,                  k(  r|dk(  rt         d|j                   dnt         d| d|j                   d}t        j                   j#                  ||      }t$        j'                  d|        t)        j*                  |      }t$        j'                  d|        n| j                  t        j.                  k(  rt         |vr)t        j                   j#                  |t         d|       n|}	t$        j'                  d|	        d	|j1                         i}t3        j4                  |t3        j6                  |	      t9               
       |d	   }t$        j'                  d|	        |j5                        }
d d d        |
S # 1 sw Y   
S xY w)Nr   r   zzSet the `sync_module_states` flag to `True` so that model states are synced across processes when initializing FSDP objectr   r   zLoading model from zModel loaded from r   r   )r   storage_readerr   )wait_for_everyoner   r   r   r   r    r!   r"   r#   r$   typer'   sync_module_states
ValueErrorr   r   r%   r&   r(   r)   r*   loadr,   r-   r   r.   load_state_dictFileSystemReaderr   )r1   r2   r   	input_dirr4   r5   r6   input_model_filer   r8   load_results              r9   load_fsdp_modelrG   P   s   !!#""m&C&CC '44q87G%%43C%%0			{**K,I,I;KnKn
 )8 &&-*G*GGE{d"{'@'@A'E"55$3  )8 )8 3>2Bj\.:,VWXcWddhHiL!ww||I|DKK-.>-?@A$45JKK,-=,>?@((M,J,JJ !# ,eK$=$=#>dC"|1[M{7P7P6QQUV 
  "ww||I|DKK-.>-?@A$45JKK,-=,>?@((M,L,LL !\)3 Y:,a}(EF 
 KK-hZ89!5#3#3#56J##%&77A*,
 $G,JKK,XJ78++J7S)8T U)8T s   AL5HLLc                    t        j                  |d       t        j                  || j                  | j                  | j
                        5  t        j                  ||      }| j                  t        j                  k(  r|j                  dk(  r&|dk(  r	t         dnt         d| d}t         j                  j                  ||      }t        j                  d|        t        j                   ||       t        j                  d|        nt         j                  j                  |t         d|       }	t        j                  |	d       t        j                  d|	        t#        j$                  d|it#        j&                  |	      t)               	       t        j                  d|	        d d d        y # 1 sw Y   y xY w)
NTr   r   r   r   zSaving Optimizer state to zOptimizer state saved in 	optimizerr   )r   r   r#   r   r    r$   optim_state_dictr   r   r'   r   r%   r&   r(   r)   r*   r+   r.   r/   r0   r   )
r1   r2   rI   r   r3   optimizer_indexoptim_stateoptim_state_nameoutput_optimizer_filer8   s
             r9   save_fsdp_optimizerrO      s   KK
T*			{**K,I,I;KnKn
 @ ++E9=&&-*G*GG((A-/>!/C~&d+NK[[\]l\mmqIr ! )+ZAQ(R%89N8OPQ

;(=>78M7NOPww||J>2B!OCT0UVHKK40KK4XJ?@##'5&77A*,
 KK3H:>?-@ @ @s   E*GG
c                 z   |j                          t        j                  || j                  | j                  | j                        5  | j                  t
        j                  k(  rd }|dk(  r	t         dnt         d| d}t        j                  j                  ||      }t        j                  d|        t        j                  |      }t        j                  d|        nt         |vr)t        j                  j                  |t         d|       n|}	t        j                  d|	        t        |j!                         dt#        j$                  |	            }|d   }t        j                  d	|	        t        j&                  |||
      }
|j)                  |
       d d d        y # 1 sw Y   y xY w)Nr   r   r   zLoading Optimizer state from zOptimizer state loaded from zLoading Optimizer from rI   )model_state_dictoptimizer_keyr<   zOptimizer loaded from )r   optimrJ   )r=   r#   r   r    r$   r   r   r   r   r%   r&   r(   r)   r*   rA   r   r   r.   rC   optim_state_dict_to_loadrB   )r1   r2   rI   r   rD   rK   rL   optimizer_nameinput_optimizer_filer8   flattened_osds              r9   load_fsdp_optimizerrX      s   !!#			{**K,I,I;KnKn
 1 &&-*G*GGK
 ,;a+?>"$'GWWXYhXiimEn  $&77<<	>#J KK78L7MNO**%9:KKK67K6LMN %%i7 Y>*:!O;L(MN 
 KK1(<=;!&!1!1!3)&77AK
 &k2KKK0
;<55Eepq!!-0=1 1 1s   E!F11F:)r   )r   r*   loggingr   	constantsr   r   r   importsr	   versionsr
   torch.distributed.checkpointdistributed
checkpointr.   ,torch.distributed.checkpoint.default_plannerr   r   &torch.distributed.checkpoint.optimizerr   2torch.distributed.fsdp.fully_sharded_data_parallelr   r#   r   __name__r(   r:   rG   rO   rX        r9   <module>rf      sd    
    G G 3 & D./4R4T22cXcP 
H	*6Z2j@6 1re   