
    ;i                        S SK r S SKrS SKrS SKrS SKrS SKJr  S SKJr  S SK	J
r
  S SKrSSKJr  SSKJrJrJrJrJrJrJrJrJrJrJrJr  SSKJr  SS	KJrJrJ r   SS
K!J"r"  SSK#J$r$J%r%  / 4S jr&S r'S\ RP                  S\)\*\*4   4S jr+S\ RP                  S\,\-\*   \)\*\*4   4   4S jr.S\ RP                  S\)\*\*4   4S jr/S\ RP                  S\,\-\*   \)\*\*4   4   4S jr0 SS\ RP                  S\)\*\*4   S\1S\,\ RP                  \)\*\*4   4   4S jjr2S\-\*   S\)\*\*4   4S jr3S\S\ RP                  S\,\ RP                  \)\*\
4   4   4S jr4S r5 " S S5      r6g)     N)literal_eval)which)Any   )SageMakerConfig)DynamoBackendPrecisionTypeis_ccl_availableis_fp8_availableis_hpu_availableis_ipex_availableis_mlu_availableis_musa_availableis_npu_availableis_sdaa_availableis_torch_xla_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)get_free_portis_port_in_usemerge_dicts)compare_versions   )DistributedTypeSageMakerDistributedTypec                     UR                  U5      u  p4[        U 5      R                  5        H0  u  pVU[        U5      R                  5       ;   d  M$  [	        X5U5        M2     U$ )z,
Filters out all `accelerate` specific args
)parse_known_argsvarsitemskeyssetattr)argsparserdefault_argsnew_args_keyvalues          g/home/dmtnaga/Documents/work/airagagent/rag_env/lib/python3.13/site-packages/accelerate/utils/launch.py_filter_argsr*   /   sU     )),7KH4j&&(
$x.%%''H5) ) O    c                      S V s/ s H  n [        U 5      (       d  M  U PM     nn [        U5      S:X  a  [        S5      eUS   n[        R                  " US/5      nSU;   a  USSSS	4$ US
SSS4$ s  sn f )z
Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs
are: OpenMPI, Intel MPI, or MVAPICH.

Returns: Program name and arg names for hostfile, num processes, and processes per node
)mpirunmpiexecr   z\mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.z	--versions   Open MPI
--hostfilez-nz
--npernodez	--bind-toz-fz-ppn )r   lenOSError
subprocesscheck_output)xmpi_appsmpi_appmpirun_versions       r)   _get_mpirun_argsr9   :   s     1=0aE!H0H=
8}tuu qkG,,g{-CDNn$dL+EE dFB.. >s
   A0A0r"   current_envc                 P   Sn[        U 5       H  nUR                  S5      (       d  M  [        X5      nUc  M+  US:X  a;  [        US   5      XS-   '   [        US   5      XS-   '   [        US   5      XS	-   '   Ml  [        [        X5      5      X UR	                  5        3'   M     U$ )
z&
Setup the FP8 environment variables.
ACCELERATE_fp8_fp8_override_linear_precisionr   FP8_OVERRIDE_FPROPr   FP8_OVERRIDE_DGRADr   FP8_OVERRIDE_WGRAD)r   
startswithgetattrstrupper)r"   r:   prefixargr(   s        r)   setup_fp8_envrH   R   s     FDz>>&!!D&E 99ADU1XK)= =>ADU1XK)= =>ADU1XK)= =><?@R<SK(399;- 89  r+   returnc                 
   / nU R                   (       a  U R                  (       a  [        S5      e[        U SS5      nU R                  nU R
                  bo  [        5       u  pEpgn[        U SS5      n	U(       a  U(       a  [        X#-  5      OSn
UUUU R
                  UU
/-  nU(       a  X[        U5      /-  nU(       a  XU	/-  nU R                   (       dA  UR                  [        R                  5        U R                  (       a  UR                  S5        UR                  U R                  5        UR                  U R                  5        [        R                  R!                  5       n[        U R"                  =(       d    U R$                  5      US'   U R&                  (       a  S	US
'   U R(                  S:w  a  U R(                  b  [+        5       (       a  U R(                  US'   O[-        5       (       a  U R(                  US'   O[/        5       (       a  U R(                  US'   Ol[1        5       (       a  U R(                  US'   OM[3        5       (       a  U R(                  US'   O.[5        5       (       a  U R(                  US'   OU R(                  US'   US:  a(  U R6                  c   S5       eU R8                  c   S5       e[;        5       (       a  [        U SS5      OSnUb  US:  d  US:  aS  U R6                  b  U R6                  OSUS'   U R8                  b  [        U R8                  5      OSUS'   [        U5      US'   US   (       a  SUS'   [        S5      US'    [=        U R>                  RA                  5       5      n[        U5      US#'   U R>                  RA                  5       S$:X  a%  [E        5       (       d  [G        S%5      e[I        X5      n [K        U RL                  RO                  5       5      nURP                  US''   U RR                  US('   [        U RT                  5      US)'   [        U RV                  5      US*'   [        U RX                  5      US+'   [        U RZ                  5      US,'   []        5       (       a&  [        U R^                  5      RA                  5       US-'   U R`                  (       a  SUS.'   X4$ ! [         a>    [        S U R>                  RA                  5        S![<        RB                  " 5        S"35      ef = f! [         a>    [        S&U RL                  RO                  5        S![J        RB                  " 5        S"35      ef = f)/zr
Prepares and returns the command list and an environment with the correct simple launcher environment variables.
0--module and --no_python cannot be used togethernum_processesNzbind-tosocket1z-mACCELERATE_USE_CPUtrueACCELERATE_DEBUG_MODEallZE_AFFINITY_MASKMLU_VISIBLE_DEVICESSDAA_VISIBLE_DEVICESMUSA_VISIBLE_DEVICESASCEND_RT_VISIBLE_DEVICESHABANA_VISIBLE_MODULESCUDA_VISIBLE_DEVICESr   zFWhen using multiple machines, you need to specify the main process IP.zHWhen using multiple machines, you need to specify the main process port.
mpirun_cclr   z	127.0.0.1MASTER_ADDR29500MASTER_PORTCCL_WORKER_COUNTzgranularity=fine,compact,1,0KMP_AFFINITYKMP_BLOCKTIMEUnknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONfp8rFP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed.Unknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMIC*ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATIONOMP_NUM_THREADSACCELERATE_USE_IPEXACCELERATE_CPU_AFFINITY)1	no_pythonmodule
ValueErrorrC   num_machinesmpirun_hostfiler9   rD   appendsys
executabletraining_scriptextendtraining_script_argsosenvironcopycpuuse_cpudebuggpu_idsr   r   r   r   r   r   main_process_ipmain_process_portr
   r	   mixed_precisionlowerlistr   RuntimeErrorrH   r   dynamo_backendrE   r(   dynamo_modedynamo_use_fullgraphdynamo_use_dynamicdynamo_use_regional_compilationnum_cpu_threads_per_processr   ipexenable_cpu_affinity)r"   cmdrL   rs   mpi_app_namehostfile_argnum_proc_argproc_per_node_argbind_to_argbind_tonproc_per_noder:   ccl_worker_countr   r   s                  r)   prepare_simple_launcher_cmd_envr   d   s    C~~$++KLLD/48M$$L'ScSePL[$	84?LQ]]:;cf  
 	
 #m"455C))C>>

3>>";;JJtJJt##$JJt(()**//#K(+DHH,D(EK$%zz/5+,||u!9.2llK*+15K-.  26,,K./  26,,K./7;||K3448LLK0126,,K./a##/ 	
T	
/ %%1 	
V	
1 :J9K9Kwt\15QR!ma&7L1<L=A=Q=Q=]T%9%9cnM"DHDZDZDfS)?)?%@lsM"*-.>*?&''(&DN#'*1vO$
'(<(<(B(B(DE 14O0DK,-!!#u,!! E  $D6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=%()I)I%JK!"-0^-A-A-C)*14-.=  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   #S #T AT
AUc                    U R                   S:X  a  [        5       U l         OU R                   c  SU l         U R                  nU R                  nU R                  nU R                   nUS:  a|  [        X-  5      U l        [        U5      U l        [        U R                  5      U l
        [        U SS5      (       a!  [        U5      U l        [        U5      U l        O0U SU 3U l        O#[        U5      U l        Ub  [        U5      U l        US:*  =(       d    [        U R                  5      S:H  nU(       aG  [        U5      (       a7  US::  a"  SU l        ["        R$                  " S	U S
35        O['        SU S35      eU R(                  (       a  U R*                  (       a  [-        S5      eU R(                  (       a  SU l        OU R*                  (       a  SU l        [.        R0                  R3                  5       nU R4                  (       a  SUS'   [        U SS5      nUS:w  a  U R6                  b|  [9        5       (       a  XvS'   Oh[;        5       (       a  XvS'   OT[=        5       (       a  XvS'   O@[?        5       (       a  XvS'   O,[A        5       (       a  XvS'   O[C        5       (       a  XvS'   OXvS'   U RD                  RG                  5       n [I        U5      n[        U5      US'   U RD                  RG                  5       S:X  a%  [M        5       (       d  [O        S5      e[Q        X5      n [S        U RT                  RW                  5       5      n	U	RX                  US '   U RZ                  US!'   [        U R\                  5      US"'   [        U R^                  5      US#'   [        U R`                  5      US$'   U Rb                  (       GaQ  SUS%'   U Rd                  (       a  U Rf                  (       d  [-        S&5      e[i        U S'5      (       a  [        U Rj                  5      OS(US)'   [        U Rl                  5      US*'   [        U Rn                  5      RG                  5       US+'   [        U Rp                  5      RG                  5       US,'   [        U Rr                  5      US-'   U Rt                  b  [        U Rt                  5      US.'   U Rv                  b  [        U Rv                  5      US/'   U Rx                  b  [        U Rx                  5      US0'   U Rz                  b  [        U Rz                  5      US1'   [        U R|                  5      RG                  5       US2'   [        U R~                  5      RG                  5       US3'   [        U Rd                  5      RG                  5       US4'   [        U Rf                  5      RG                  5       US5'   [        U R                  5      RG                  5       US6'   [        U S7S5      b  [        U R                  5      US8'   U R                  (       a  S9n
SUS:'   [        U R                  5      XjS;-   '   [        U R                  5      XjS<-   '   [        U R                  5      XjS=-   '   U R                  b  [        U R                  5      XjS>-   '   U R                  b  [        U R                  5      XjS?-   '   U R                  b  [        U R                  5      XjS@-   '   U R                  b  [        U R                  5      XjSA-   '   [        U R                  5      USB'   U R                  (       a  S(USC'   U R                  (       d  U$ SDn
U R                  (       a  SUSE'   [        U R                  5      XjSF-   '   [        U R                  5      XjSG-   '   [        U R                  5      XjSH-   '   [        U R                  5      XjSI-   '   U R                  S:  a  [        U R                  5      XjSJ-   '   U$ ! [,         a&    [-        SU S[H        RJ                  " 5        S35      ef = f! [,         a>    [-        SU RT                  RW                  5        S[R        RJ                  " 5        S35      ef = f)KzW
Prepares and returns an environment with the correct multi-GPU environment variables.
r   N<s  r   same_networkF:TPort `  ` is already in use. Accelerate will attempt to launch in a standalone-like mode by finding an open port automatically for this session. If this current attempt fails, or for more control in future runs, please specify a different port (e.g., `--main_process_port <your_chosen_port>`) or use `--main_process_port 0` for automatic selection in your launch command or Accelerate config file.3Tried to launch distributed communication on port `$  `, but another process is utilizing it. Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.rK   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   ACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`fsdp_versionrN   FSDP_VERSIONFSDP_SHARDING_STRATEGYFSDP_RESHARD_AFTER_FORWARDFSDP_OFFLOAD_PARAMSFSDP_MIN_NUM_PARAMSFSDP_AUTO_WRAP_POLICYFSDP_TRANSFORMER_CLS_TO_WRAPFSDP_BACKWARD_PREFETCHFSDP_STATE_DICT_TYPEFSDP_FORWARD_PREFETCHFSDP_USE_ORIG_PARAMSFSDP_CPU_RAM_EFFICIENT_LOADINGFSDP_SYNC_MODULE_STATESFSDP_ACTIVATION_CHECKPOINTINGfsdp_ignored_modulesFSDP_IGNORED_MODULESMEGATRON_LM_ACCELERATE_USE_MEGATRON_LM	TP_DEGREE	PP_DEGREEGRADIENT_CLIPPINGNUM_MICRO_BATCHESSEQUENCE_PARALLELISMRECOMPUTE_ACTIVATIONSUSE_DISTRIBUTED_OPTIMIZERrm   ro   PARALLELISM_CONFIG_!ACCELERATE_USE_PARALLELISM_CONFIGDP_REPLICATE_SIZETP_SIZECP_SIZEDP_SHARD_SIZECP_COMM_STRATEGY)Rr   r   rL   rs   r   rD   r   nnodesintmachine_rank	node_rankrC   master_addrmaster_portrdzv_endpointr   
standalonewarningswarnConnectionErrorrq   rp   rr   r{   r|   r}   r   r   r   r   r   r   r   r   r   r   r	   r   r   r   rH   r   r   rE   r(   r   r   r   r   use_fsdpfsdp_cpu_ram_efficient_loadingfsdp_sync_module_stateshasattrr   fsdp_sharding_strategyfsdp_reshard_after_forwardfsdp_offload_paramsfsdp_min_num_paramsfsdp_auto_wrap_policy"fsdp_transformer_layer_cls_to_wrapfsdp_backward_prefetchfsdp_state_dict_typefsdp_forward_prefetchfsdp_use_orig_paramsfsdp_activation_checkpointingr   use_megatron_lmmegatron_lm_tp_degreemegatron_lm_pp_degreemegatron_lm_gradient_clippingmegatron_lm_num_micro_batches megatron_lm_sequence_parallelism!megatron_lm_recompute_activations%megatron_lm_use_distributed_optimizerr   r   use_parallelism_config$parallelism_config_dp_replicate_sizeparallelism_config_tp_sizeparallelism_config_cp_size parallelism_config_dp_shard_size#parallelism_config_cp_comm_strategy)r"   rL   rs   r   r   need_port_checkr:   r   r   r   rF   s              r)   prepare_multi_gpu_envr      s   
 "!.				'!&&&M$$L**O..a!-"?@,'T../4//"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'F3t/@/@+AQ+FO>*;<<1"DOMM*+ ,D D "EFWEX Y   {{t~~KLL		**//#Kzz/5+,dIu-G%DLL4.5*+18-.  29./  29./7>344;0129./**002Ou'8 14O0DK,-!!#u,!! E  $D6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=}}}-3)*..t7S7Svww@Gn@]@]c$*;*;&<cfN# 14D4O4O0P,-478W8W4X4^4^4`01-01I1I-J-P-P-R)*-01I1I-J)*%%136t7Q7Q3RK/022>:=d>e>e:fK67&&2478S8S4TK01$$025d6O6O2PK.//243M3M/N/T/T/V+,.1$2K2K.L.R.R.T*+8;D<_<_8`8f8f8h4514T5Q5Q1R1X1X1Z-.7:4;];]7^7d7d7f344/6B25d6O6O2PK./4:01,/0J0J,K[(),/0J0J,K[()478Z8Z4[001--98;D<^<^8_K!44500<;>t?d?d;eK!77811=<?@f@f<gK!88955A@CDDnDn@oK!<<=%()I)I%JK!"14-.&&"F"";A78478a8a4b001*-d.M.M*NY&'*-d.M.M*NY&'03D4Y4Y0Z_,-**Q.7:4;c;c7dK!334k  u9/9JJ[\i\n\n\p[qqrsttu  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   _= "#`0 =0`-0Aa8c                    U R                   S:X  a  [        5       U l         OU R                   c  SU l         U R                  nU R                  nU R                  nU R                   nSnU R
                  c  [        S   U l        US:  Ga@  U R
                  [        S   :w  Ga(  S/nUR                  S[        U R                  5      /5        U R
                  S:X  aE  [        SSS	5      (       a  [        S
5      eUR                  S[        U R                  5      S/5        O'UR                  SS[        U R
                  5      /5        U R                  b'  UR                  S[        U R                  5      /5        OgU R                  b'  UR                  S[        U R                  5      /5        O3UR                  S[        U R                  U R                  -  5      /5        U(       a  UR                  S[        U5      /5        UR                  S[        U5      /5        U R                  (       a  U R                   (       a  [        S5      eU R                  (       a  UR#                  S5        O"U R                   (       a  UR#                  S5        UR#                  U R$                  5        UR                  U R&                  5        OUS:  a  U R
                  [        S   :X  a|  [        X-  5      U l        [        U5      U l        [-        U R                  5      U l        [1        U SS5      (       a!  [        U5      U l        [        U5      U l        O0U SU 3U l        O#[        U5      U l        Ub  [        U5      U l        US:*  =(       d    [-        U R                  5      S:H  nU(       aG  [9        U5      (       a7  US::  a"  SU l        [<        R>                  " SU S35        O[A        SU S35      eU R                  (       a  U R                   (       a  [        S5      eU R                  (       a  SU l        OU R                   (       a  SU l        [B        RD                  RG                  5       nU RH                  (       a  SUS '   [1        U S!S"5      nUS":w  a  U RJ                  b|  [M        5       (       a  XS#'   Oh[O        5       (       a  XS$'   OT[Q        5       (       a  XS%'   O@[S        5       (       a  XS&'   O,[U        5       (       a  XS''   O[W        5       (       a  XS('   OXS)'    [Y        U RZ                  R]                  5       5      n	[a        S-[B        Rb                  Re                  S,5      5      US-'   [        U	5      US.'   U RZ                  R]                  5       S/:X  a%  [g        5       (       d  [i        S05      e[k        X5      n[        U Rl                  5      R]                  5       US1'   SUS2'   U Rn                  b  [        U Rn                  5      US3'   U Rp                  b  [        U Rp                  5      US4'   U Rr                  b&  [        U Rr                  5      R]                  5       US5'   U Rt                  b&  [        U Rt                  5      R]                  5       US6'   U Rv                  b&  [        U Rv                  5      R]                  5       US7'   U Rx                  b&  [        U Rx                  5      R]                  5       US8'   U Rz                  b&  [        U Rz                  5      R]                  5       US9'   U R|                  b  [        U R|                  5      US:'   U R~                  (       a  S;US<'   U R                  b  [        U R                  5      US='   XW4$ ! [         a>    [        S*U RZ                  R]                  5        S+[X        R^                  " 5        S,35      ef = f)>zl
Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
r   Nr   r   	deepspeedr/   nossh<z0.14.5z+nossh launcher requires DeepSpeed >= 0.14.5z--node_rankz--no_sshz--no_local_rankz
--launcherz	--excludez	--includez
--num_gpusz--master_addrz--master_portrK   z--modulez--no_pythonr   Fr   Tr   r   r   r   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   ra   rb   rc   
PYTHONPATHrd   re   rf   ACCELERATE_CONFIG_DS_FIELDSACCELERATE_USE_DEEPSPEEDACCELERATE_DEEPSPEED_ZERO_STAGE&ACCELERATE_GRADIENT_ACCUMULATION_STEPSACCELERATE_GRADIENT_CLIPPING-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEACCELERATE_DEEPSPEED_ZERO3_INIT+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL ACCELERATE_DEEPSPEED_CONFIG_FILErN   ro   (ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES)Ar   r   rL   rs   r   deepspeed_multinode_launcherr   ry   rD   deepspeed_hostfiler   rr   r   deepspeed_exclusion_filterdeepspeed_inclusion_filterrq   rp   ru   rx   rz   r   r   r   r   rC   r   r   r   r   r   r   r   r   r{   r|   r}   r   r   r   r   r   r   r   r   r	   r   r   r   env_var_path_addpathabspathr   r   rH   'deepspeed_fields_from_accelerate_config
zero_stagegradient_accumulation_stepsgradient_clippingoffload_optimizer_deviceoffload_param_devicezero3_init_flagzero3_save_16bit_modeldeepspeed_config_filer   deepspeed_moe_layer_cls_names)
r"   rL   rs   r   r   r   r   r:   r   r   s
             r)   prepare_deepspeed_cmd_envr  q  s   
 "!.				'!&&&M$$L**O..
C ((0,I!,L)aD==A^_`Aaam

L#d&=&=">?@,,7S(;; !NOOJJs4+<+<'=zJKJJ)<T=^=^9_`a**6JJ778 ,,8JJ778 JJc$*<*<@Q@Q*Q&RSTJJ_)=>?

OS):%;<=;;4>>OPP[[JJz"^^JJ}%

4''(

4,,-		d??C`abCcc!-"?@,'T../4//"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'F3t/@/@+AQ+FO>*;<<1"DOMM*+ ,D D "EFWEX Y   {{t~~KLL		**//#Kzz/5+,dIu-G%DLL4.5*+18-.  29./  29./7>344;0129./
'(<(<(B(B(DE !1rwws?S TK03O0DK,-!!#u,!! E  $D614T5a5a1b1h1h1jK-..4K*+"9<T__9M56''3@CDDdDd@e<=)69$:P:P6Q6W6W6Y23$$0GJ4KhKhGiGoGoGqCD  ,CFtG`G`CaCgCgCi?@'9<T=Q=Q9R9X9X9Z56"".EHIdIdEeEkEkEmAB!!-:=d>X>X:Y6714-.))5BEdFhFhBi>?G  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

s   &#^ A_%podc                     U R                   S:X  a*  [        SS9(       a  U R                  (       a  SUS'   OSUS'   U R                  (       a  SUS'   U(       a"  U R                  U l        U R                  U l        X4$ )	zQ
Prepares and returns an environment with the correct TPU environment variables.
bf16T)check_is_tpurN   XLA_DOWNCAST_BF16XLA_USE_BF16rP   rQ   )r   r   downcast_bf16r   tpu_vmvmtpu_nametpu)r"   r:   r  s      r)   prepare_tpur    sl     v%*@d*S/2K+,*-K'zz/5+,
++==r+   nargsc                 N   [        U 5      S:  a  0 $ S n[        R                  " 5       nUR                  U 5      u  p4[	        U5       H  u  pVUR                  S5      (       d  M  S nUS-   [        U5      :  a'  XES-      R                  S5      (       a  [        S5      eO[        S5      eUc  UR                  XaS9  Mw  UR                  XgS9  M     UR                  U 5      R                  R                  5        VV	s0 s H  u  pXS;   a  [        U	5      OU	_M     sn	n$ s  sn	nf )	Nr   c                 j     [        U 5      n U S-  U :X  a  [        U 5      $ U $ ! [         a    U s $ f = f)Nr   )floatr   rr   )ss    r)   _infer_type+_convert_nargs_to_dict.<locals>._infer_type(  s=    	aAAv{1vH 	H	s   # # 22)-z--r   ul   SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)action)TrueFalse)r1   argparseArgumentParserr   	enumeraterB   rr   add_argument
parse_args__dict__r   r   )
r  r!  r#   r&   unknownindexargumentr%  r'   r(   s
             r)   _convert_nargs_to_dictr1  #  s3   
5zA~	 $$&F((/JA$W-{++Fqy3w<'19%00==$ G  > ! C  ~##H#?##H#<# .* !++E2;;AACCJC 	.?%?l5!UJC  s   >D!sagemaker_configc                 "
   [        S5        U R                  [        R                  S'   U R                  b  U R                  [        R                  S'   O`UR
                  bH  UR                  b;  UR
                  [        R                  S'   UR                  [        R                  S'   O[        S5      e[        R                  R                  UR                  5      nU(       d  Sn[        R                  R                  UR                  5      nUR                  S5      (       d  [        S	U S
35      e[        S5        [        UR                  5      n [!        UR"                  R%                  5       5      n [)        UR*                  R-                  5       5      nS[/        U5      UR0                  UR2                  [/        UR4                  5      [/        UR6                  5      [/        UR8                  5      U R:                  R0                  S.nUR"                  R%                  5       S:X  a%  [=        5       (       d  [?        S5      e[A        X5      nS nU R:                  [B        RD                  :X  a  SSSS000nS n	U RF                  b  [        SU RF                   S35        0 n	[I        U RF                  5       n
[K        U
5       H6  u  pUS:X  a  M  URM                  S5      nUS   RO                  5       XS   '   M8     S S S 5        [        SU	 35        S nU RP                  b  [        SU RP                   S35        / n[I        U RP                  5       n
[K        U
5       HH  u  pUS:X  a  M  URM                  S5      nUS   US   RO                  5       S.nURS                  U5        MJ     S S S 5        [        SU 35        [        S 5        U RT                  UUU RV                  U RX                  U RZ                  U R\                  U R^                  U R`                  U Rb                  S!UUUUS".nU Rd                  b  [g        U Rd                  U5      nX4$ ! [         a>    [        SUR"                  R%                  5        S[         R&                  " 5        S35      ef = f! [         a>    [        SUR*                  R-                  5        S[(        R&                  " 5        S35      ef = f! , (       d  f       GN= f! , (       d  f       GNX= f)#Nz(Configuring Amazon SageMaker environmentAWS_DEFAULT_REGIONAWS_PROFILEAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profilerc   z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersra   rb   rg   rP   )ACCELERATE_USE_SAGEMAKERrd   rh   ri   rj   rk   rl   %ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPEre   rf   smdistributeddataparallelenabledTzLoading SageMaker Inputs from z filer   	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )NameRegexzLoaded SageMaker Metrics: zCreating EstimatorF)	image_urientry_point
source_dirroletransformers_versionpytorch_version
py_versionbase_job_nameinstance_countinstance_typedebugger_hook_configdistributionhyperparametersenvironmentmetric_definitions)4printregionr{   r|   profileaws_access_key_idaws_secret_access_keyr2   r  dirnamerx   basenameendswithrr   r1  rz   r	   r   r   r   r   r   rE   rD   r(   r   r   r   r   distributed_typer   r   rH   r   DATA_PARALLELsagemaker_inputs_fileopenr*  splitstripsagemaker_metrics_fileru   rA  iam_role_namerE  rF  rG  rH  rs   ec2_instance_typeadditional_argsr   )r2  r"   rC  rB  rM  r   r   rN  rL  sagemaker_inputsfileilinelsagemaker_metricsmetric_dicts                   r)   prepare_sagemager_args_inputsri  M  s    

45'7'>'>BJJ#$ +$4$<$<

=!				+0J0J0V*.*@*@

&'.2.H.H

*+uvv !5!56J
''""4#7#78K&&ST_S``abcc	
34,T-F-FGO
'(<(<(B(B(DE
&t':':'@'@'BC %+&)/&:%3%9%9"&"2"2+.t/H/H+I),T-D-D)E69$:^:^6_1A1R1R1X1X	K !!#u,!! E  $D6L((,D,R,RR'.9d:K)LM --9./?/U/U.VV[\]"889T$T?6JJt$)*1 1&	 + : 	)*:);<= ..:/0@0W0W/XX]^_"99:d$T?6JJt$aDqTZZ\ "((5 + ; 	*+<*=>? 

%//"  .. 0 E E+;;&11)77*77);; %$*"/D$ ''3+;;TB!!m  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

@ :9 ;:s4   #Q 9#R" ;AS-AS?AR"AS*-
S<?
Tc                    [         R                  R                  U S5      R                  S5       Vs/ s H  n[	        U5      S:  d  M  UPM     nnUR                  [        U5      5        SR                  U5      $ s  snf )z
Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
caller to set it in os.environ.
r0   r   r   )r{   r|   getr\  r1   ru   rD   join)env_var_namepath_to_addppathss       r)   r  r    sd    
 

|R8>>sCRC1s1vPQzQCER	LL[!"88E? Ss   A<	A<c                   (    \ rS rSrSrSS jrS rSrg)PrepareForLaunchi  aE  
Prepare a function that will launched in a distributed setup.

Args:
    launcher (`Callable`):
        The function to launch.
    distributed_type ([`~state.DistributedType`]):
        The distributed type to prepare for.
    debug (`bool`, *optional*, defaults to `False`):
        Whether or not this is a debug launch.
c                 <    Xl         [        U5      U l        X0l        g )N)launcherr   rX  r   )selfrt  rX  r   s       r)   __init__PrepareForLaunch.__init__  s      /0@ A
r+   c                 x   U R                   (       a  [        [        R                  R	                  S5      5      n[        R                  R	                  S5      n[
        R                  R                  SU[
        R                  R                  XC5      US9  OU R                  [        R                  [        R                  [        R                  [        R                  [        R                  [        R                   4;   a  [#        U5      [        R                  S'   [        [        R                  R	                  SS5      5      n[        [        R                  R	                  SS	5      5      n[#        XV-  U-   5      [        R                  S
'   [#        S5      [        R                  S'   U R$                  " U6   g )N
WORLD_SIZEACCELERATE_DEBUG_RDV_FILEgloo)rankstore
world_size
LOCAL_RANKNPROCr   	NODE_RANKr   RANKFORK_LAUNCHED)r   r   r{   r|   rk  torchdistributedinit_process_group	FileStorerX  r   	MULTI_GPU	MULTI_MLU
MULTI_MUSA	MULTI_NPU	MULTI_XPU	MULTI_CPUrD   rt  )ru  r/  r"   r~  rdv_filenprocr   s          r)   __call__PrepareForLaunch.__call__  s3   ::RZZ^^L9:Jzz~~&ABH00''11(G%	 1  ""%%%%&&%%%%%%'
 
 (+5zBJJ|$

w23EBJJNN;:;I!$U%6%>!?BJJv&)!f

?#tr+   )r   rX  rt  N)NOF)__name__
__module____qualname____firstlineno____doc__rv  r  __static_attributes__ r+   r)   rr  rr    s    

r+   rr  )F)7r(  r{   r3   rv   r   astr   shutilr   typingr   r  commands.config.config_argsr   utilsr   r	   r
   r   r   r   r   r   r   r   r   r   utils.constantsr   utils.otherr   r   r   utils.versionsr   dataclassesr   r   r*   r9   	NamespacedictrD   rH   tupler   r   r   r  boolr  r1  ri  r  rr  r  r+   r)   <module>r     s    	  
      9    < D D - B -/ /0** c3h $d(*<*< dtCyRVWZ\_W_R`G`Aa dNc 2 2 ctCH~ cL[H$6$6 [5cDQTVYQYNAZ;[ [~ HM


+/S>@D
8tCH~-.('$s) 'S#X 'Ts"%s"-5-?-?s"
8tCH~-.s"l+ +r+   