9i\_ <ddlZddlZddlZddlmZddlmZmZmZm Z ddl Z ddl m Z ddl mZmZmZmZmZddlmZddlmZmZd d lmZmZgfd Zd ej8d e eeeeefffdZd ej8d eeeffdZd ej8d e eeeeefffdZ dd ej8deeefde!d e ej8eeefffdZ"deed eeeffdZ#de d ej8d e ej8eeefffdZ$dZ%GddZ&y)N) literal_eval)AnyDictListTuple)SageMakerConfig) DynamoBackend PrecisionTypeis_ipex_availableis_npu_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)is_port_in_use merge_dicts)DistributedTypeSageMakerDistributedTypec|j|\}}t|jD].\}}|t|jvs"t |||0|S)z4 Filters out all `accelerate` specific args )parse_known_argsvarsitemskeyssetattr)argsparser default_argsnew_args_keyvalues a/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/accelerate/utils/launch.py _filter_argsr#$s`)),7KHa4j&&(* U $x.%%' ' Hc5 )* Orreturncg}|jr|jr td|js<|jtj |jr|jd|j|j |j|jtjj}t|jxs |j|d<|jrd|d<|j dk7rO|j Ct#r|j |d<n)t%r|j |d<n|j |d <|j&d kDr(|j(|d <t|j*|d <nR|j,d kDrC|j( |j(nd |d <|j*t|j*nd|d < t/|j0j3}t||d< t7|j8j;}|j<|d<|j>|d<t|j@|d<t|jB|d<t|jD|d<tGrLt|jHj3|d<t|jJj3|d<||fS#t$r=td|j0j3dt/j4dwxYw#t$r=td|j8j;dt7j4dwxYw)zz Prepares and returns the command list and an environment with the correct simple launcher environment variables. 0--module and --no_python cannot be used togetherz-mACCELERATE_USE_CPUtrueACCELERATE_DEBUG_MODEallZE_AFFINITY_MASKASCEND_RT_VISIBLE_DEVICESCUDA_VISIBLE_DEVICESr MASTER_ADDR MASTER_PORTz 127.0.0.129500Unknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONUnknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMICOMP_NUM_THREADSACCELERATE_USE_IPEXACCELERATE_USE_XPU)& no_pythonmodule ValueErrorappendsys executabletraining_scriptextendtraining_script_argsosenvironcopystrcpuuse_cpudebuggpu_idsrr num_machinesmain_process_ipmain_process_port num_processesr mixed_precisionlowerlistr dynamo_backendupperr! dynamo_modedynamo_use_fullgraphdynamo_use_dynamicnum_cpu_threads_per_processr ipexuse_xpu)rcmd current_envrSrVs r"prepare_simple_launcher_cmd_envr`/s C ~~$++KLL >> 3>>" ;; JJt JJt##$JJt(()**//#K(+DHH,D (EK$% zz/5 +, ||u!9  .2llK* +  7;||K3 426,,K. / 1%)%9%9 M"%()?)?%@ M"  a =A=Q=Q=]T%9%9cn M"DHDZDZDfS)?)?%@ls M" '(<(<(B(B(DE 14O0DK,- &t':':'@'@'BC 0>/C/CK+,,0,<,|d(<t|j@j%|d)<t|jB|d*<|jDt|jD|d+<|jFt|jF|d,<|jHt|jH|d-<|jJt|jJ|d.<t|jLj%|d/<t|jNj%|d0<t|j:j%|d1<t|j<j%|d2<|jPrd3} d|d4<t|jR|| d5z<t|jT|| d6z<t|jV|| d7z<|jXt|jX|| d8z<|jZt|jZ|| d9z<|j\t|j\|| d:z<|j^t|j^|| d;z<t|j`|d<<|S#t$r%td|dt'j(dwxYw#t$r=td!|j,j/dt+j(dwxYw)=z_ Prepares and returns an environment with the correct multi-GPU environment variables. rRrOrPrQrnproc_per_nodennodes node_rank same_networkF master_addr master_port rdzv_endpoint:r)r*rNr+r,r-r.r2r3r4r5r6r7r8r9r:ACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`FSDP_SHARDING_STRATEGYFSDP_OFFLOAD_PARAMSFSDP_MIN_NUM_PARAMSFSDP_AUTO_WRAP_POLICYFSDP_TRANSFORMER_CLS_TO_WRAPFSDP_BACKWARD_PREFETCHFSDP_STATE_DICT_TYPEFSDP_FORWARD_PREFETCHFSDP_USE_ORIG_PARAMSFSDP_CPU_RAM_EFFICIENT_LOADINGFSDP_SYNC_MODULE_STATES MEGATRON_LM_ACCELERATE_USE_MEGATRON_LM TP_DEGREE PP_DEGREEGRADIENT_CLIPPINGNUM_MICRO_BATCHESSEQUENCE_PARALLELISMRECOMPUTE_ACTIVATIONSUSE_DISTRIBUTED_OPTIMIZERr;)1getattrrrJint machine_rankrConnectionErrorr?r>r@rGrHrIrMrNrr rSrTr rUr rVrWr!rXrYrZuse_fsdpfsdp_cpu_ram_efficient_loadingfsdp_sync_module_statesfsdp_sharding_strategyfsdp_offload_paramsfsdp_min_num_paramsfsdp_auto_wrap_policy"fsdp_transformer_layer_cls_to_wrapfsdp_backward_prefetch_policyfsdp_state_dict_typefsdp_forward_prefetchfsdp_use_orig_paramsuse_megatron_lmmegatron_lm_tp_degreemegatron_lm_pp_degreemegatron_lm_gradient_clippingmegatron_lm_num_micro_batches megatron_lm_sequence_parallelism!megatron_lm_recompute_activations%megatron_lm_use_distributed_optimizerr[) rrRrOrPrQr_rNrSrVprefixs r"prepare_multi_gpu_envrjsD/2M40Ld$56O&9:a&M\,I(JKhL 12k3t'8'8#9: 4 / D-_)= > D-->)? @ D/o->a@Q?R+S T&M(:;  ( D-->)? @ !'(ABSATU{ {   {{t~~KLL h% k4(**//#K zz/5 +,dIu-G%DLL4  .5K* +  7>K3 429K. /**002Ou'814O0DK,- &t':':'@'@'BC 0>/C/CK+,,0,<,:=d>e>e:fK6 7  - - 9478Z8Z4[K0 1  $ $ 025d6O6O2PK. //243M3M/N/T/T/V +,.1$2K2K.L.R.R.T *+8;D<_<_8`8f8f8h 4514T5Q5Q1R1X1X1Z -. 4: 01,/0J0J,K F[(),/0J0J,K F[()478Z8Z4[ F001  - - 98;D<^<^8_K!44 5  0 0 <;>t?d?d;eK!77 8  1 1 =r)r*rNr+r.r,r2r3r4 PYTHONPATHr5ACCELERATE_CONFIG_DS_FIELDSACCELERATE_USE_DEEPSPEEDACCELERATE_DEEPSPEED_ZERO_STAGE&ACCELERATE_GRADIENT_ACCUMULATION_STEPSACCELERATE_GRADIENT_CLIPPING-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEACCELERATE_DEEPSPEED_ZERO3_INIT+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL ACCELERATE_DEEPSPEED_CONFIG_FILE)+rrrrrErJdeepspeed_hostfiledeepspeed_exclusion_filterdeepspeed_inclusion_filterrRrOr?r>r@rArDrFrrrrrGrHrIrMrNrr rSrTrUenv_var_path_addpathabspath'deepspeed_fields_from_accelerate_config zero_stagegradient_accumulation_stepsgradient_clippingoffload_optimizer_deviceoffload_param_devicezero3_init_flagzero3_save_16bit_modeldeepspeed_config_file) rrRrOrPrQr^r_rNrSs r"prepare_deepspeed_cmd_envrs_D/2M40Ld$56O&9: C ((046STU6VWaD==A^_`Aaa-. L#d&=&="> cRVRsRsNtuv  * * 6 JJ778   , , 8 JJ778  JJ c$*<*<@Q@Q*Q&RS T OS):%;<= ;;4>>OP P [[ JJz " ^^ JJ} % 4''( 4,,-  d??C`abCcc&M\,I(JKhL 12k3t'8'8#9: 4 / D-_)= > D-->)? @ D/o->a@Q?R+S T&M(:;  ( D-->)? @ !'(ABSATU{ {   {{t~~KLL h% k4(**//#K zz/5 +,dIu-G%DLL4!29K. /.5K* + '(<(<(B(B(DE !1rwws?S TK 03O0DK,-14T5a5a1b1h1h1jK-..4K*+ "9X>X:Y 67  3  ,T-A-A-G-G-I,JJ[\i\n\n\p[qqr s   s 5#V''AW-r_podc|jdk(r|jrd|d<nd|d<|jrd|d<|r"|j|_|j |_||fS)zY Prepares and returns an environment with the correct TPU environment variables. bf161XLA_DOWNCAST_BF16 XLA_USE_BF16r)r*)rS downcast_bf16rMtpu_vmvmtpu_nametpu)rr_rs r" prepare_tpurHsf  v%   /2K+ ,*-K ' zz/5 +, ++==  r$nargsc8t|dkriSd}tj}|j|\}}t |D]\}}|j dsd}|dzt|kr"||dzj drt dt d||j||n|j|||j|jjD cic]\}} || dvr t| n| c} }Scc} }w) Nrcf t|}|dz|k(r t|S|S#t$r|cYSwxYw)Nr)floatrr@)ss r" _infer_typez+_convert_nargs_to_dict.._infer_typeas= aAAv{1v H H s"" 00)-z--rulSageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)action)TrueFalse) lenargparseArgumentParserr enumerate startswithr@ add_argument parse_args__dict__rr) rrrrunknownindexargumentrr r!s r"_convert_nargs_to_dictr\s6 5zA~  $ $ &F((/JAw$W-=x   { +Fqy3w<'519%00=$G!C~##H;#?##HV#<#=*!++E2;;AAC  C U.?%?l5!UJ  s6Dsagemaker_configc2 td|jtjd<|j|jtjd<n^|j G|j ;|j tjd<|j tjd<n tdtjj|j}|sd}tjj|j}|jdstd |d td t|j} t!|j"j%} t)|j*j-}dt/||j0|j2t/|j4t/|j6|j8j0d}d}|j8t:j<k(rddddiii}d} |j>td|j>di} tA|j>5} tC| D]5\} } | dk(r | jEd} | djG| | d<7 dddtd| d}|jHtd|jHdg}tA|jH5} tC| D]F\} } | dk(r | jEd} | d| djGd}|jK|H dddtd|td|jL|||jN|jP|jR|jT|jV|jX|jZd||||d }|j\t_|j\|}|| fS#t$r=td |j"j%d t!j&dwxYw#t$r=td|j*j-d t)j&dwxYw#1swYxYw#1swYJxYw)!Nz(Configuring Amazon SageMaker environmentAWS_DEFAULT_REGION AWS_PROFILEAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profiler4z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersr2r3r6r))ACCELERATE_USE_SAGEMAKERr5r7r8r9r:%ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPE smdistributed dataparallelenabledTzLoading SageMaker Inputs from z filer rzLoaded SageMaker Inputs: zLoading SageMaker Metrics from )NameRegexzLoaded SageMaker Metrics: zCreating EstimatorF) image_uri entry_point source_dirroletransformers_versionpytorch_version py_version base_job_nameinstance_count instance_typedebugger_hook_config distributionhyperparameters environmentmetric_definitions)0printregionrGrHprofileaws_access_key_idaws_secret_access_keyEnvironmentErrorrdirnamerDbasenameendswithr@rrFr rSrTrUr rVrWrJr!rXrYrZdistributed_typer DATA_PARALLELsagemaker_inputs_fileopenrsplitstripsagemaker_metrics_filerAr iam_role_namerrrrrOec2_instance_typeadditional_argsr)rrrrrrSrVrrsagemaker_inputsfileilinelsagemaker_metrics metric_dicts r"prepare_sagemager_args_inputsrs 45'7'>'>BJJ#$+$4$<$< =!   +0J0J0V*.*@*@ &'.2.H.H *+ k  !5!56J  ''""4#7#78K    &ST_S``abcc 34,T-F-FGO '(<(<(B(B(DE  &t':':'@'@'BC%+&)/&:%3%9%9"&"2"2+.t/H/H+I),T-D-D)E1A1R1R1X1XKL((,D,R,RR'.9d:K)LM --9 ./?/U/U.VV[\] "88 9 6T$T? 646JJt$)*1 1&  6 6 )*:);<=..: /0@0W0W/XX]^_ "99 : 6d$T? 646JJt$aDqTZZ\ "((5 6 6 *+<*=>? %//"  .. 0 E E+;;&11)77*77);; %$*"/ D$''3+;;TB ! !!_  ,T-A-A-G-G-I,JJ[\i\n\n\p[qqr s     &t':':'@'@'B&CCTUbUgUgUiTjjk l   2 6 6 6 6s4#O-,#P6AQ?.AR -AP36AQ<?R  Rctjj|djdDcgc]}t |dkDs|}}|j t |dj|Scc}w)z Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the caller to set it in os.environ. rir)rGrHgetrrrArJjoin) env_var_name path_to_addppathss r"rrsa  |R8>>sC R1s1vPQzQ RE R LL[!" 88E? Ss A7A7ceZdZdZddZdZy)PrepareForLaunchai Prepare a function that will launched in a distributed setup. Args: launcher (`Callable`): The function to launch. distributed_type ([`~state.DistributedType`]): The distributed type to prepare for. debug (`bool`, *optional*, defaults to `False`): Whether or not this is a debug launch. c@||_t||_||_y)N)launcherrrrM)selfr!rrMs r"__init__zPrepareForLaunch.__init__s  /0@ A r$c8|jrttjj d}tjj d}t j jd|t j j|||n|jtjtjtjtjfvrt|tjd<ttjj dd}ttjj dd }t||z|ztjd <tdtjd <|j |y) N WORLD_SIZEACCELERATE_DEBUG_RDV_FILEgloo)rankstore world_size LOCAL_RANKNPROCr NODE_RANKrRANK FORK_LAUNCHED)rMrrGrHrtorch distributedinit_process_group FileStorerr MULTI_GPU MULTI_NPU MULTI_XPU MULTI_CPUrJr!)r"rrr*rdv_filenprocrds r"__call__zPrepareForLaunch.__call__s' ::RZZ^^L9:Jzz~~&ABH    0 0''11(JG% 1   " "  % %  % %  % %  % % '  (+5zBJJ| $ w23EBJJNN;:;I!$UY%6%>!?BJJv &)!f ?# tr$N)NOF)__name__ __module__ __qualname____doc__r#r:r$r"rrs  r$r)F)'rrGrBastrtypingrrrrr0commands.config.config_argsr utilsr r r r rutils.constantsr utils.otherrr dataclassesrrr# NamespacerJr`rrboolrrrrrr@r$r"rJs )) 9<5B-/8(*<*<8tCyRVWZ\_W_R`G`Aa8vj 2 2jtCH~jZnH$6$6n5cDQTVYQYNAZ;[ndHM   +/S>@D 8  tCH~ -.('$s)'S#X'Tn"%n"-5-?-?n" 8  tCH~ -.n"b))r$