9iddlZddlZddlmZddlmZmZmZmZddl Z ddl Z ddl m Z ddZdZdZd eeej$fd eee j&ffd ZGd d eZGddeZd eee j&fdeefdZy)N)Mapping)DictListOptionalUnion)is_safetensors_availablecd}t|jdk(r!|jtj}d}|j j }tjj||d}|2|t|j}|t|jd||<|jdk(r|d}tj||jd|j}|dd|dd|j|S)Nztorch.bfloat16bfloat16.dat)dtypeshaperzw+)r moder)strr viewtorchint16cpunumpyospathjoinlistrndimnpmemmapflush)weight weight_nameoffload_folderindexr array tensor_file file_arrays b/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/accelerate/utils/offload.pyoffload_weightr&s E 6<<,,U[[) JJL   E'',,~+d/CDK  = $E',tEKK7HIk zzQd ;ekkEKKXJ!HJqM Lct|d}|dk(rd}|d}|dk(rd}tj|||d}t|dd k(r|d }t j |}|ddk(r|j tj}|S) Nr)rr r rr)r rrr)tuplerrlenrtensorrr ) weight_file weight_inforr rs r%load_offloaded_weightr0/s +g& 'E {  E  YY{%u3 GF ;w  A% \\& !F7z)U^^, Mr'c|t|dk(rytjj|d}tjj |r-t |dd5}t j|}dddni}j|t |dd5}t j||ddddy#1swYJxYw#1swYyxYw) Nr index.jsonr*zutf-8)encodingw)indent) r,rrrisfileopenjsonloadupdatedump)r!r offload_index_filef current_indexs r%save_offload_indexr@Es }E anlC ww~~() $cG < ) IIaLM ) )   # 8.A -1-..  ) ) ..sCC C  Csave_dir state_dictctj|di}|jD]\}}t||||}t ||y)a Offload a state dict in a given folder. Args: save_dir (`str` or `os.PathLike`): The directory in which to offload the state dict. state_dict (`Dict[str, torch.Tensor]`): The dictionary of tensors to offload. T)exist_ok)r!N)rmakedirsitemsr&r@)rArBr!name parameters r%offload_state_dictrIVsQKK4( E%++-Giy$FGuh'r'c2eZdZdZdedefdZdZdZdZ y) PrefixedDatasetz Will access keys in a given dataset by adding a prefix. Args: dataset (`Mapping`): Any map with string keys. prefix (`str`): A prefix to add when trying to access any element in the underlying dataset. datasetprefixc ||_||_yNrLrM)selfrLrMs r%__init__zPrefixedDataset.__init__rs  r'c<|j|j|SrOrPrQkeys r% __getitem__zPrefixedDataset.__getitem__vs||t{{mC5122r'ct|jDcgc] }|j|js|"c}Scc}wrO)iterrL startswithrMrTs r%__iter__zPrefixedDataset.__iter__ys.DLLPSCNN4;;4OSPQQPs AAc,t|jSrO)r,rLrQs r%__len__zPrefixedDataset.__len__|s4<<  r'N) __name__ __module__ __qualname____doc__rrrRrVrZr]r)r'r%rKrKis*3R!r'rKceZdZdZ d deeejfdee ee jfde fdZ defdZd Zd Zy) OffloadedWeightsLoaderaC A collection that loads weights stored in a given state dict or memory-mapped on disk. Args: state_dict (`Dict[str, torch.Tensor]`, *optional*): A dictionary parameter name to tensor. save_folder (`str` or `os.PathLike`, *optional*): The directory in which the weights are stored (by `offload_state_dict` for instance). index (`Dict`, *optional*): A dictionary from weight name to their information (`dtype`/ `shape` or safetensors filename). Will default to the index saved in `save_folder`. NrB save_folderr!c|| | td|in||_||_|I|Gttj j |d5}tj|}ddd|in||_ t|jj|_ |jj|jDcgc]}||jvs|c}||_y#1swYxYwcc}w)NzWNeed either a `state_dict`, a `save_folder` or an `index` containing offloaded weights.r2) ValueErrorrBrdr8rrrr9r:r!rkeysall_keysextenddevice)rQrBrdr!rjr>rUs r%rRzOffloadedWeightsLoader.__init__s  +"5%-vw w * 2" & =[4bggll; => %! !  % =Re T__1134  TZZTc3dmm;ScTU  % %UsC,C8C8,C5rUc0||jvr|j|S|j|}|jdts t dddlm}|jdn |j}||dd|5}|j|jd|}dddd |vr"jtt|d SStjj|j|d }t!||S#1swYjxYw) Nsafetensors_filezRThese offloaded weights require the use of safetensors: `pip install safetensors`.r) safe_openrpt) frameworkrjrr r )rBr!getr ImportError safetensorsrmrj get_tensortogetattrrrrrrdr0)rQrUr/rmrjr>r-r.s r%rVz"OffloadedWeightsLoader.__getitem__s $// !??3' 'jjo ??- . :+-!"vww -"kk1Ut{{F;'9:dSYZ K^_koomS&IJ K+%yy G0D!EFF ggll4#3#3uD\B $[+>> K Ks "D  Dc,t|jSrO)rXrhr\s r%rZzOffloadedWeightsLoader.__iter__sDMM""r'c,t|jSrO)r,rhr\s r%r]zOffloadedWeightsLoader.__len__s4==!!r')NNNN)r^r_r`rarrrTensorrrrPathLikerrRrVrZr]r)r'r%rcrcsm /39= ell*+eC$456 (?s?.#"r'rcsubmodule_namesc i}|D]N}|j|jDcic]!\}}||k(s|j|dzr||#c}}P|Scc}}w)a Extract the sub state-dict corresponding to a list of given submodules. Args: state_dict (`Dict[str, torch.Tensor]`): The state dict to extract from. submodule_names (`List[str]`): The list of submodule names we want to extract. .)r;rFrY)rBrzresult module_namerUparams r%extract_submodules_state_dictrssF&    #-"2"2"4 C+% c8I)JU      M  s&A rO)r9rcollections.abcrtypingrrrrrrrimportsr r&r0r@rryrxrIrKrcrr)r'r%rs #.. -*,."(sBKK'7!8(d3PUP\P\K\F](&!g!.="W="@d3 3D.EX\]`Xar'