
    ;i?                     r   S SK r S SKJr  S SKJrJr  S SKrS SKJr  SSK	J
r
  SSKJrJrJrJrJr  SSKJrJrJr  SSKJr  SS	KJr  SS
KJr  / SQr " S S5      r " S S\5      rS)S\R<                  S\S\4S jjr S)S\R<                  4S jjr! " S S\5      r"   S*S\R                  R<                  S\\#\$\RJ                  4   S\\\$\&\$   4      S\\&\$      S\\'\#\'\RJ                  \RP                  4   4      4
S jjr)        S+S\R                  R<                  S\\RJ                     S\S\\   S\S\$S\\\$\&\$   4      S\\&\$      S\\'\#\'\RJ                  \RP                  4   4      4S  jjr*S\R<                  4S! jr+        S+S\R<                  S\\\RJ                  \'\$\RJ                  4   4      S\\\'\$\4   4   S\\   S\S\$S\\\$\&\$   4      S\\&\$      S\\'\#\'\RJ                  \RP                  4   4      4S" jjr, " S# S$\5      r- " S% S&5      r. " S' S(\5      r/g),    N)Mapping)OptionalUnion   )PartialState)PrefixedDatasetfind_devicenamed_module_tensorssend_to_deviceset_module_tensor_to_device)is_mlu_availableis_musa_availableis_npu_available)clear_device_cache)get_non_persistent_buffers)recursive_getattr)tocudanpuxpumlusdaamusac                   4    \ rS rSrSrSrS rS rS rS r	Sr
g	)
	ModelHook+   ag  
A hook that contains callbacks to be executed just before and after the forward method of a model. The difference
with PyTorch existing hooks is that they get passed along the kwargs.

Class attribute:
- **no_grad** (`bool`, *optional*, defaults to `False`) -- Whether or not to execute the actual forward pass under
  the `torch.no_grad()` context manager.
Fc                     U$ )z
To be executed when the hook is attached to the module.

Args:
    module (`torch.nn.Module`): The module attached to this hook.
 selfmodules     `/home/dmtnaga/Documents/work/airagagent/rag_env/lib/python3.13/site-packages/accelerate/hooks.py	init_hookModelHook.init_hook7   	         c                     X#4$ )a  
To be executed just before the forward method of the model.

Args:
    module (`torch.nn.Module`): The module whose forward pass will be executed just after this event.
    args (`Tuple[Any]`): The positional arguments passed to the module.
    kwargs (`Dict[Str, Any]`): The keyword arguments passed to the module.

Returns:
    `Tuple[Tuple[Any], Dict[Str, Any]]`: A tuple with the treated `args` and `kwargs`.
r   r    r!   argskwargss       r"   pre_forwardModelHook.pre_forward@   s     |r&   c                     U$ )a  
To be executed just after the forward method of the model.

Args:
    module (`torch.nn.Module`): The module whose forward pass been executed just before this event.
    output (`Any`): The output of the module.

Returns:
    `Any`: The processed `output`.
r   r    r!   outputs      r"   post_forwardModelHook.post_forwardN   s	     r&   c                     U$ )z
To be executed when the hook is detached from a module.

Args:
    module (`torch.nn.Module`): The module detached from this hook.
r   r   s     r"   detach_hookModelHook.detach_hook[   r%   r&   r   N)__name__
__module____qualname____firstlineno____doc__no_gradr#   r+   r0   r3   __static_attributes__r   r&   r"   r   r   +   s"     Gr&   r   c                   6    \ rS rSrSrS rS rS rS rS r	Sr
g	)
SequentialHooke   zP
A hook that can contain several hooks and iterates through them at each event.
c                     Xl         g Nhooks)r    rB   s     r"   __init__SequentialHook.__init__j   s    
r&   c                 N    U R                    H  nUR                  U5      nM     U$ r@   )rB   r#   r    r!   hooks      r"   r#   SequentialHook.init_hookm   s#    JJD^^F+F r&   c                 ^    U R                    H  nUR                  " U/UQ70 UD6u  p#M     X#4$ r@   )rB   r+   )r    r!   r)   r*   rG   s        r"   r+   SequentialHook.pre_forwardr   s6    JJD++FDTDVDLD& |r&   c                 N    U R                    H  nUR                  X5      nM     U$ r@   )rB   r0   )r    r!   r/   rG   s       r"   r0   SequentialHook.post_forwardw   s%    JJD&&v6F r&   c                 N    U R                    H  nUR                  U5      nM     U$ r@   )rB   r3   rF   s      r"   r3   SequentialHook.detach_hook|   s%    JJD%%f-F r&   rA   N)r5   r6   r7   r8   r9   rC   r#   r+   r0   r3   r;   r   r&   r"   r=   r=   e   s     


r&   r=   r!   rG   appendc                 2   U(       a0  [        U SS5      b"  U R                  n[        U 5        [        X15      n[	        U S5      (       a  [	        U S5      (       a  U R
                  nOU R                  nX@l        UR                  U 5      n Xl        S nS[        [        U 5      5      ;   a<  [        R                  " [        R                  " XP5      U5      U R                  l        U $ [        R                  " [        R                  " XP5      U5      U l        U $ )a  
Adds a hook to a given module. This will rewrite the `forward` method of the module to include the hook, to remove
this behavior and restore the original `forward` method, use `remove_hook_from_module`.

<Tip warning={true}>

If the module already contains a hook, this will replace it with the new hook passed by default. To chain two hooks
together, pass `append=True`, so it chains the current and new hook into an instance of the `SequentialHook` class.

</Tip>

Args:
    module (`torch.nn.Module`):
        The module to attach a hook to.
    hook (`ModelHook`):
        The hook to attach.
    append (`bool`, *optional*, defaults to `False`):
        Whether the hook should be chained with an existing one (if module already contains a hook) or not.

Returns:
    `torch.nn.Module`: The same module, with the hook attached (the module is modified in place, so the result can
    be discarded).
_hf_hookN_old_forwardc                 \   U R                   R                  " U /UQ70 UD6u  pU R                   R                  (       a1  [        R                  " 5          U R                  " U0 UD6nS S S 5        OU R                  " U0 UD6nU R                   R                  U W5      $ ! , (       d  f       N*= fr@   )rQ   r+   r:   torchrR   r0   )r!   r)   r*   r/   s       r"   new_forward'add_hook_to_module.<locals>.new_forward   s    226KDKFK??"",,d=f= ! (($9&9F++FF;;	 !s   B
B+GraphModuleImpl)getattrrQ   remove_hook_from_moduler=   hasattrrR   forwardr#   strtype	functoolsupdate_wrapperpartial	__class__)r!   rG   rO   old_hookold_forwardrU   s         r"   add_hook_to_modulerd      s    0 76:t4@??'h-vz""wv~'F'F))nn)^^F#FO< CV--#,#;#;I<M<Mk<bdo#p  M #11)2C2CK2XZefMr&   c                    [        U S5      (       a'  U R                  R                  U 5        [        U S5        [        U S5      (       aQ  S[	        [        U 5      5      ;   a  U R                  U R                  l        OU R                  U l        [        U S5        [         H  nU R                  R                  US5        M!     U(       a"  U R                  5        H  n[        X15        M     U $ )ad  
Removes any hook attached to a module via `add_hook_to_module`.

Args:
    module (`torch.nn.Module`): The module to attach a hook to.
    recurse (`bool`, **optional**): Whether to remove the hooks recursively

Returns:
    `torch.nn.Module`: The same module, with the hook detached (the module is modified in place, so the result can
    be discarded).
rQ   rR   rW   N)rZ   rQ   r3   delattrr\   r]   rR   ra   r[   _accelerate_added_attributes__dict__popchildrenrY   )r!   recurseattrchilds       r"   rY   rY      s     vz""##F+
#v~&& DL 11'-':':F$#00FN' -D$' - __&E#E3 ' Mr&   c                       \ rS rSrSr        SS\\\\\	R                  4      S\S\S\\   S\S	\S
\\\\\   4      S\\\\\	R                  \	R                  4   4      4S jjrS rS rS rS rS rSrg)AlignDevicesHook   aK  
A generic `ModelHook` that ensures inputs and model weights are on the same device for the forward pass of the
associated module, potentially offloading the weights after the forward pass.

Args:
    execution_device (`torch.device`, *optional*):
        The device on which inputs and model weights should be placed before the forward pass.
    offload (`bool`, *optional*, defaults to `False`):
        Whether or not the weights should be offloaded after the forward pass.
    io_same_device (`bool`, *optional*, defaults to `False`):
        Whether or not the output should be placed on the same device as the input was.
    weights_map (`Mapping[str, torch.Tensor]`, *optional*):
        When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.
    offload_buffers (`bool`, *optional*, defaults to `False`):
        Whether or not to include the associated module's buffers when offloading.
    place_submodules (`bool`, *optional*, defaults to `False`):
        Whether to place the submodules on `execution_device` during the `init_hook` event.
Nexecution_deviceoffloadio_same_deviceweights_mapoffload_buffersplace_submodules	skip_keystied_params_mapc	                     Xl         X l        X0l        X@l        XPl        X`l        Xpl        S U l        0 U l        0 U l	        [        5       U l        Xl        g r@   )rq   rr   rs   rt   ru   rv   rw   input_deviceparam_original_devicesbuffer_original_devicessettied_params_namesrx   )	r    rq   rr   rs   rt   ru   rv   rw   rx   s	            r"   rC   AlignDevicesHook.__init__   sX     !1,&. 0" !&(#')$!$  /r&   c                     SU R                    SU R                   SU R                   SU R                   SU R                   S[        U R                  5       S3$ )Nz"AlignDevicesHook(execution_device=z
, offload=z, io_same_device=z, offload_buffers=z, place_submodules=z, skip_keys=))rq   rr   rs   ru   rv   reprrw   r    s    r"   __repr__AlignDevicesHook.__repr__  sn    01F1F0GzRVR^R^Q_ `"1122DTEYEYDZ [  $ 5 56l4CWBXXY[	
r&   c                    U R                   S:X  d$  U R                   [        R                  " S5      :X  a  S U l        U R                  (       dJ  U R                   b=  [        XR                  S9 H$  u  p#[        XU R                   U R                  S9  M&     U$ U R                  (       Ga  [        XR                  S9 VVs0 s H  u  p$X$R                  _M     snnU l        U R                  cG  [        XR                  U R                  S9 VVs0 s H  u  p$X$R                  S5      _M     snnU l	        [        XR                  U R                  SS9 H`  u  p#U R                  bB  [        X5      R                  5       U R                  ;   a  U R                  R                  U5        [        XS5        Mb     U R                  (       dP  U R                   bC  UR!                  U R                  S9 H$  u  p#[        XU R                   U R                  S9  M&     U$ U R                  (       aF  U R                   b9  [#        XR                  S9 H"  n[        XU R                   U R                  S9  M$     U$ s  snnf s  snnf )Nmeta)rk   )rx   )include_buffersrk   cpuTr   rk   remove_non_persistent)rq   rT   devicerx   rr   r
   rv   r   original_devicesrt   ru   r   r   data_ptrr~   addnamed_buffersr   )r    r!   name_params        r"   r#   AlignDevicesHook.init_hook  s/     F*d.C.Cu||TZG[.[#'D || 5 5 A/@U@UV+F$:O:Oaeauauv WN K \\\6J6[p[p6q%6q{tll"6q%D! ' (<0D0DdNcNc($( ((5/)($  0(<(<dF[F[sw ((4)&7@@BdFZFZZ**..t4+F&A ''D,A,A,M%33D<Q<Q3RGD/d&;&;TMaMa  S  %%$*?*?*K6vG\G\]D/d&;&;TMaMa ^
 I%$s   ?I1I7c           
      <   U R                   (       a  [        X#/5      U l        U R                  (       Ga  [	        5       U l        [        UU R                  U R                  SS9 GH  u  pES nU R                  U   nSU;   ak  UR                  SS5      U R                  R                  5       ;   a=  UR                  [        R                  :X  a  U R                  UR                  SS5         nX@R                  ;   a;  UR!                  5       U R"                  ;  a  0 U R"                  UR!                  5       '   Ub  U R"                  b~  UR!                  5       U R"                  ;   a`  U R$                  U R"                  UR!                  5          ;  a5  U R
                  R'                  UR!                  5       U R$                  45        [)        UUU R$                  UUU R"                  S9  GM     [+        X R$                  5      [+        X0R$                  U R,                  S94$ )NTr   weightSCB)valuefp16_statisticsrx   rw   )rs   r	   rz   rr   r}   tied_pointers_to_remover
   ru   rv   rt   replacekeysdtyperT   int8r~   r   rx   rq   r   r   r   rw   )r    r!   r)   r*   r   r   r   r   s           r"   r+   AlignDevicesHook.pre_forwardH  s    +TN ;D<<<+.5D(/ $ 4 4--&*	 #'((.t#Xu(EIYIYI^I^I`(`{{ejj0*.*:*:4<<RW;X*Y
 111enn6FdNbNb6b=?D(()9: %,,8(D,@,@@--T5I5I%..JZ5[[0044enn6FH]H]5^_+))$3$($8$85F d$9$9:N))T^^=
 
 	
r&   c                    U R                   (       Ga  [        UU R                  U R                  SS9 HN  u  p4[	        XS5        [        U5      R                  S:X  d  M,  S UR                  l        S UR                  l	        MP     U R                   H|  u  pV[        U[        5      (       a>  [        5       (       a  SU 3nO)[        5       (       a  SU 3nO[        5       (       a  SU 3nX`R                   U   ;   d  Ml  U R                   U   U	 M~     [#        5       U l
        U R$                  (       a+  U R&                  b  [)        X R&                  U R*                  S9nU$ )	NTr   r   Linear8bitLtznpu:zmlu:zmusa:r   )rr   r
   ru   rv   r   r]   r5   stater   CxBr   
isinstanceintr   r   r   rx   r}   rs   rz   r   rw   )r    r!   r/   r   r   value_pointerr   s          r"   r0   AlignDevicesHook.post_forwardu  s)   <<</ $ 4 4--&*	 ,F&A<((N:'+FLL$'+FLL$ *.)E)E%fc**'))#'x)++#'x*,,#(!111-@@,,];FC *F ,/5D(4#4#4#@#F,=,=XFr&   c           
          U R                   (       ac  U R                  R                  5        HE  u  p#U[        R                  " S5      :w  d  M!  [        XX0R                  R                  US 5      S9  MG     U$ )Nr   )r   )rr   r   itemsrT   r   r   rt   get)r    r!   r   r   s       r"   r3   AlignDevicesHook.detach_hook  s[    << $ 5 5 ; ; =U\\&11/fL\L\L`L`aegkLlm !> r&   )r|   rq   rz   rs   rr   ru   r   r{   rv   rw   rx   r~   r   rt   )NFFNFFNN)r5   r6   r7   r8   r9   r   r   r   r\   rT   r   boolr   listdictTensorrC   r   r#   r+   r0   r3   r;   r   r&   r"   ro   ro      s    * EI$)- %!&59QU/"5c5<<)?#@A/ / 	/
 g&/ / / E#tCy.12/ "$sDu||1K,L'L"MN/:
-^+
Z>r&   ro   rq   rw   preload_module_classesrx   c           	         [        U S5      (       d1  [        U R                  5       5      S:  a  [        U [	        XUS95        Ub  U R
                  R                  U;   a  gU R                  5        H  n[        UUUUUS9  M     g)a^  
Recursively attaches `AlignDevicesHook` to all submodules of a given model to make sure they have the right
execution device

Args:
    module (`torch.nn.Module`):
        The module where we want to attach the hooks.
    execution_device (`int`, `str` or `torch.device`):
        The device on which inputs and model weights should be placed before the forward pass.
    skip_keys (`str` or `List[str]`, *optional*):
        A list of keys to ignore when moving inputs or outputs between devices.
    preload_module_classes (`List[str]`, *optional*):
        A list of classes whose instances should load all their weights (even in the submodules) at the beginning
        of the forward. This should only be used for classes that have submodules which are registered but not
        called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
        `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
    tied_params_map (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], *optional*, defaults to `None`):
        A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution
        device, this parameter is useful to reuse the first available pointer of a shared weight for all others,
        instead of duplicating memory.
rQ   r   rw   rx   N)rw   r   rx   )	rZ   len
state_dictrd   ro   ra   r5   rj   attach_execution_device_hook)r!   rq   rw   r   rx   rm   s         r"   r   r     s    8 6:&&3v/@/@/B+Ca+G-Tcd	
 )f.>.>.G.GKa.a"$#9+	
 #r&   rr   rt   ru   module_namec	                    [        U 5      n	U=(       a$    USL=(       a    U R                  R                  U;   n
[        [	        U	5      5      S:  d  U
(       a@  Ub"  [        U5      S:  a  U S3OSn[        X;5      nOSn[        UUUUU
UUS9n[        XSS9  U
(       a  gU R                  5        H.  u  p[        U5      S:  a  U SU 3OUn[        UUUUUUUUUS9	  M0     g)	am  
Recursively attaches `AlignDevicesHook` to all submodules of a given model that have direct parameters and/or
buffers.

Args:
    module (`torch.nn.Module`):
        The module where we want to attach the hooks.
    execution_device (`torch.device`, *optional*):
        The device on which inputs and model weights should be placed before the forward pass.
    offload (`bool`, *optional*, defaults to `False`):
        Whether or not the weights should be offloaded after the forward pass.
    weights_map (`Mapping[str, torch.Tensor]`, *optional*):
        When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.
    offload_buffers (`bool`, *optional*, defaults to `False`):
        Whether or not to include the associated module's buffers when offloading.
    module_name (`str`, *optional*, defaults to `""`):
        The name of the module.
    skip_keys (`str` or `List[str]`, *optional*):
        A list of keys to ignore when moving inputs or outputs between devices.
    preload_module_classes (`List[str]`, *optional*):
        A list of classes whose instances should load all their weights (even in the submodules) at the beginning
        of the forward. This should only be used for classes that have submodules which are registered but not
        called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
        `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
    tied_params_map (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], *optional*, defaults to `None`):
        A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution
        device, this parameter is useful to reuse the first available pointer of a shared weight for all others,
        instead of duplicating memory.
Nr   . )rq   rr   rt   ru   rv   rw   rx   T)rO   rq   rr   rt   ru   r   r   rw   rx   )
r
   ra   r5   r   r   r   ro   rd   named_childrenattach_align_device_hook)r!   rq   rr   rt   ru   r   rw   r   rx   directsfull_offloadprefixprefixed_weights_maprG   
child_namerm   s                   r"   r   r     s   R #6*Gn*$6n6;K;K;T;TXn;n  4=A"*-k*:Q*>}A&BF#2;#G #' -,+)+
 	65  $224
69+6F6J}Aj\2PZ
 -#+"#9+
	
 5r&   c                 ^    [        U 5        U R                  5        H  n[        U5        M     g)z
Recursively removes all hooks attached on the submodules of a given model.

Args:
    module (`torch.nn.Module`): The module on which to remove all hooks.
N)rY   rj   remove_hook_from_submodules)r!   rm   s     r"   r   r     s%     F#"#E* #r&   c	                    [        U[        5      (       dF  [        U[        5      (       d1  U(       d  [        USUSUS9n	[	        X	5        g[        U USUUUUUS9  g[        U[        5      (       d   UR                  5        V
s0 s H  oU_M     nn
[        U[        5      (       d   UR                  5        V
s0 s H  oU_M     nn
XQ;   a:  XR;   a5  X%   (       d,  [        X   UUS:H  SUUS9n	[	        X	5        [        XU   XhS9  OXQ;   aU  XR;   aP  [        U X   SUUUUUUS9	  [        U S	5      (       d  [        X   US:H  UUS
9n	[	        X	5        [        U X   UUUS9  O,US:X  a&  [        UR                  S5      SUUS
9n	[	        X	5        U R                  5        H.  u  p[        U5      S:  a  U SU 3OUn[        UUUUUUUUUS9	  M0     gs  sn
f s  sn
f )a  
Attaches `AlignDevicesHook` to all blocks of a given model as needed.

Args:
    module (`torch.nn.Module`):
        The module where we want to attach the hooks.
    execution_device (`torch.device` or `Dict[str, torch.device]`, *optional*):
        The device on which inputs and model weights should be placed before the forward pass. It can be one device
        for the whole module, or a dictionary mapping module name to device.
    offload (`bool`, *optional*, defaults to `False`):
        Whether or not the weights should be offloaded after the forward pass. It can be one boolean for the whole
        module, or a dictionary mapping module name to boolean.
    weights_map (`Mapping[str, torch.Tensor]`, *optional*):
        When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.
    offload_buffers (`bool`, *optional*, defaults to `False`):
        Whether or not to include the associated module's buffers when offloading.
    module_name (`str`, *optional*, defaults to `""`):
        The name of the module.
    skip_keys (`str` or `List[str]`, *optional*):
        A list of keys to ignore when moving inputs or outputs between devices.
    preload_module_classes (`List[str]`, *optional*):
        A list of classes whose instances should load all their weights (even in the submodules) at the beginning
        of the forward. This should only be used for classes that have submodules which are registered but not
        called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
        `dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
    tied_params_map (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], *optional*, defaults to `None`):
        A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution
        device, this parameter is useful to reuse the first available pointer of a shared weight for all others,
        instead of duplicating memory.
T)rq   rs   rw   rv   rx   )rq   rr   rt   ru   r   rw   rx   Nr   )rq   ru   rs   rv   rw   rx   r   )rq   rr   rt   ru   r   rw   r   rx   rQ   )rq   rs   rw   rx   )r   rw   rx   r   r   r   )r   r   r   ro   rd   r   r   r   rZ   r   r   r   "attach_align_device_hook_on_blocks)r!   rq   rr   rt   ru   r   rw   r   rx   rG   keyr   rm   s                r"   r   r   +  s7   T &00GT9R9R#!1##!% /D v, 	 %!1' /'# /	 	&00=D\\^L^c!11^Lgw''+;+@+@+BC+BC<+BC&;+A'J^-:+'2-!+
 	6($[1Y	
 
	([-C -:#+##9+
	
 vz**#!1!> +r 1# /	D v,$)#9+	
 
	-11"5+	
 	6(#224
69+6F6J}Aj\2PZ
*-#+"#9+
	
 5k MCs   G8Gc                   j    \ rS rSrSr  SS\\\\\	R                  4      S\S   4S jjrS rS	 rS
rg)
CpuOffloadi  a  
Offloads a model on the CPU until its forward pass is called. The model will not be offloaded back to the CPU after
the forward, the user needs to call the `init_hook` method again for this.

Args:
    execution_device(`str`, `int` or `torch.device`, *optional*):
        The device on which the model should be executed. Will default to the MPS device if it's available, then
        GPU 0 if there is a GPU, and finally to the CPU.
    prev_module_hook (`UserCpuOffloadHook`, *optional*):
        The hook sent back by [`cpu_offload_with_hook`] for a previous model in the pipeline you are running. If
        passed, its offload method will be called just before the forward of the model to which this hook is
        attached.
Nrq   prev_module_hookUserCpuOffloadHookc                 V    X l         Ub  Xl        g [        5       R                  U l        g r@   )r   r   default_devicerq   )r    rq   r   s      r"   rC   CpuOffload.__init__  s'    
 !14D4P 0VbVdVsVsr&   c                 $    UR                  S5      $ Nr   )r   r   s     r"   r#   CpuOffload.init_hook  s    yyr&   c                 <   U R                   b  [        U R                   [        5      (       aw  U R                   R                  n[	        UR                  5       5      R                  nU[        R                  " S5      :w  a$  U R                   R                  5         [        5         [	        UR                  5       5      R                  nX`R                  :X  a  X#4$ UR                  U R                  5        [        X R                  5      [        X0R                  5      4$ r   )r   r   r   modelnext
parametersr   rT   rr   r   rq   r   r   )r    r!   r)   r*   prev_moduleprev_devicecurrent_devices          r"   r+   CpuOffload.pre_forward  s      ,D<Q<QSe1f1f//55K{5578??K ell511%%--/"$ f//1299222<		$''(d$9$9:N6ShSh<iiir&   )rq   r   )NN)r5   r6   r7   r8   r9   r   r   r\   r   rT   r   rC   r#   r+   r;   r   r&   r"   r   r     sQ      EI;?t"5c5<<)?#@At ##78t jr&   r   c                   *    \ rS rSrSrS rS rS rSrg)r   i  z
A simple hook grouping a model and a `ModelHook`, which provides easy APIs for to call the init method of the hook
or remove it entirely.
c                     Xl         X l        g r@   )r   rG   )r    r   rG   s      r"   rC   UserCpuOffloadHook.__init__  s    
	r&   c                 N    U R                   R                  U R                  5        g r@   )rG   r#   r   r   s    r"   rr   UserCpuOffloadHook.offload  s    		DJJ'r&   c                 .    [        U R                  5        g r@   )rY   r   r   s    r"   removeUserCpuOffloadHook.remove  s    

+r&   )rG   r   N)	r5   r6   r7   r8   r9   rC   rr   r   r;   r   r&   r"   r   r     s    
(,r&   r   c                       \ rS rSrSrSrS\R                  S\R                  S\SS4S	 jr	S
\R                  R                  4S jrS
\R                  R                  4S jrS
\R                  R                  4S jrSrg)LayerwiseCastingHooki  z
A hook that casts the weights of a module to a high precision dtype for computation, and to a low precision dtype
for storage. This process may lead to quality loss in the output, but can significantly reduce the memory
footprint.
Fstorage_dtypecompute_dtypenon_blockingreturnNc                 (    Xl         X l        X0l        g r@   )r   r   r   )r    r   r   r   s       r"   rC   LayerwiseCastingHook.__init__  s    **(r&   r!   c                 N    UR                  U R                  U R                  S9  U$ N)r   r   r   r   r   r   s     r"   r#   LayerwiseCastingHook.init_hook  #    		**9J9J	Kr&   c                 P    UR                  U R                  U R                  S9  X#4$ r   )r   r   r   r(   s       r"   r+    LayerwiseCastingHook.pre_forward  s&    		**9J9J	K|r&   c                 N    UR                  U R                  U R                  S9  U$ r   r   r.   s      r"   r0   !LayerwiseCastingHook.post_forward  r   r&   )r   r   r   )r5   r6   r7   r8   r9   _is_statefulrT   r   r   rC   nnModuler#   r+   r0   r;   r   r&   r"   r   r     su     L)ekk )%++ )]a )fj )
 %((// 588?? r&   r   )F)NNN)NFNFr   NNN)0r^   collections.abcr   typingr   r   rT   torch.nnr   r   r   utilsr   r	   r
   r   r   utils.importsr   r   r   utils.memoryr   utils.modelingr   utils.otherr   rg   r   r=   r   r   rd   rY   ro   r   r\   r   r   r   r   r   r   r   r   r   r   r   r   r&   r"   <module>r      s+    # "     
 - 6 *  S 7 7tY :7ryy 7	 74 7t"BII "Jxy x| 2626MQ-
HHOO-
Cell23-
 c49n-.-
 %T#Y/	-

 d3U\\5<<-G(H#HIJ-
d 04%)!1526MQP
HHOOP
u||,P
 P
 '"	P

 P
 P
 c49n-.P
 %T#Y/P
 d3U\\5<<-G(H#HIJP
f	+		 	+ PT,1%)!1526MQC
IIC
uU\\4U\\8I3J%JKLC
 4c4i()C
 '"	C

 C
 C
 c49n-.C
 %T#Y/C
 d3U\\5<<-G(H#HIJC
L+j +j\, ,"9 r&   