
    o9iOb                         d dl mZ d dl mZ d dlmZ d dlmZ d dlZd dl	m
Z  G d d      Z G d d	      Z G d
 dej                  j                        Z G d de      Z G d de      Zy)    )abc)defaultdict)deepcopy)chainNc                       e Zd Zd Zy)MockArgsc                 2    |D ]  }t        | |||           y N)setattr)selfinitial_datakeys      f/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/bitsandbytes/optim/optimizer.py__init__zMockArgs.__init__   s"     	2CD#|C01	2    N)__name__
__module____qualname__r    r   r   r   r      s    2r   r   c                   B    e Zd ZdZd Zd Zed        Zd Z	 ddZ	d Z
y)	GlobalOptimManagerNc                     t        d      )NzCall get_instance() instead)RuntimeErrorr   s    r   r   zGlobalOptimManager.__init__   s    899r   c                 J    i | _         i | _        d | _        d| _        g | _        y )NF)
pid2configindex2config	optimizeruses_config_overridemodule_weight_config_tripler   s    r   
initializezGlobalOptimManager.initialize   s(    $)!+-(r   c                     | j                   0| j                  |       | _         | j                   j                          | j                   S r
   )	_instance__new__r!   )clss    r   get_instancezGlobalOptimManager.get_instance"   s6    == KK,CMMM$$&}}r   c                    t        |      }t        |d   t              sd|ig}t        |      D ]Z  \  }}t        |d         D ]D  \  }}t	        |      | j
                  v s| j
                  t	        |         | j                  ||f<   F \ y )Nr   params)list
isinstancedict	enumerateidr   r   )r   r(   param_groupsgroup_indexgroupp_indexps          r   register_parametersz&GlobalOptimManager.register_parameters)   s    F|,q/40%|45L"+L"9 	K'h8 
a5DOO+@D1AD%%{G&<=	r   c                 t   d| _         t        |t        j                  j                        r|g}t        |t        j
                        r|g}|
||J ||i}|_|D ]Y  }t        |      | j                  v r(| j                  t        |         j                  |       B|| j                  t        |      <   [ yy)a  
        Overrides initial optimizer config for specific parameters.

        The key-values of the optimizer config for the input parameters are overridden
        This can be both, optimizer parameters like "betas", or "lr" or it can be
        8-bit specific parameters like "optim_bits", "percentile_clipping".

        Parameters
        ----------
        parameters : torch.Tensor or list(torch.Tensors)
            The input parameters.
        key : str
            The hyperparamter to override.
        value : object
            The value for the hyperparamters.
        key_value_dict : dict
            A dictionary with multiple key-values to override.
        TN)	r   r*   torchnn	ParameterTensorr-   r   update)r   
parametersr   valuekey_value_dictr2   s         r   override_configz"GlobalOptimManager.override_config5   s    * %)!j%(("4"45$Jj%,,/$J?u0!)))!5\N% <a5DOO+OOBqE*11.A-;DOOBqE*	< &r   c                 @    | j                   j                  |||f       y r
   )r    append)r   module
param_nameconfigs       r   register_module_overridez+GlobalOptimManager.register_module_overrideZ   s    ((//V0LMr   )NNN)r   r   r   r#   r   r!   classmethodr&   r3   r=   rC   r   r   r   r   r      s=    I:.  
 @D#<JNr   r   c                        e Zd Zd fd	Zd Z fdZd Zd Zd Z e	j                         dd       Zd Zd	 Zd
 Ze	j                  fdZd Z xZS )Optimizer8bitc                    t         |   ||       d| _        i | _        || _        t
        j                  j                         | _        t        j                         | _
        h d| _        |dk(  r| j                          y y )NF>   max1max2qmap1qmap2state1state2absmax1absmax2new_max1new_max2	gnorm_vec	unorm_vec   )superr   initialized	name2qmapis_pagedFGlobalPageManagerr&   page_mngr   mngnon_castable_tensor_keys	fill_qmap)r   r(   defaults
optim_bitsrX   	__class__s        r   r   zOptimizer8bit.__init___   sp    *  ++88:%224)
% ?NN r   c                     t        j                  d      | j                  d<   t        j                  d      | j                  d<   y )NT)signeddynamicFudynamic)rY   create_dynamic_maprW   r   s    r   r^   zOptimizer8bit.fill_qmapy   s4    $%$8$8$Ey!%&%9%9%Gz"r   c                 $    t         |   |       y r
   )rU   __setstate__)r   statera   s     r   rh   zOptimizer8bit.__setstate__}   s    U#r   c                     t        |      } j                  }|d   }t        |      t        |      k7  rt        d      d |D        }d |D        }t	        d t        ||      D              rt        d      t        t        j                  d |D              t        j                  d |D                    D ci c]  \  }}||
 }}} fd	t        t              }	|d
   j                         D ]   \  }
}|
|v r||
   } ||      |	|<   ||	|
<   " d }t        ||      D cg c]  \  }} |||       }}} j                  |	|d       yc c}}w c c}}w )zLoads the optimizer state.

        Args:
            state_dict (dict): optimizer state. Should be an object returned
                from a call to :meth:`state_dict`.
        r.   z<loaded state dict has a different number of parameter groupsc              3   8   K   | ]  }t        |d            ywr(   Nlen.0gs     r   	<genexpr>z0Optimizer8bit.load_state_dict.<locals>.<genexpr>   s     71c!H+&7   c              3   8   K   | ]  }t        |d            ywrl   rm   ro   s     r   rr   z0Optimizer8bit.load_state_dict.<locals>.<genexpr>   s     =1c!H+&=rs   c              3   ,   K   | ]  \  }}||k7    y wr
   r   )rp   p_lens_lens      r   rr   z0Optimizer8bit.load_state_dict.<locals>.<genexpr>   s     N,%u~Ns   z]loaded state dict contains a parameter group that doesn't match the size of optimizer's groupc              3   &   K   | ]	  }|d      ywrl   r   ro   s     r   rr   z0Optimizer8bit.load_state_dict.<locals>.<genexpr>   s     #FAAhK#F   c              3   &   K   | ]	  }|d      ywrl   r   ro   s     r   rr   z0Optimizer8bit.load_state_dict.<locals>.<genexpr>   s     #@AAhK#@ry   c                     t        |t        j                        rJ j                         r8|j                  t        j
                  k7  r|j                   j                        }|S t        |t              rS|j                         D ]>  \  }}|j                  v r|j                   j                        ||<   3  |      ||<   @ |S t        |t        j                        r t        |       fd|D              S |S )zBMake a deep copy of value, casting all tensors to device of param.c              3   0   K   | ]  } |        y wr
   r   )rp   vcastparams     r   rr   z>Optimizer8bit.load_state_dict.<locals>.cast.<locals>.<genexpr>   s     "Aa4q>"As   )r*   r5   r8   is_floating_pointdtypeuint8tor+   itemsr]   devicecontainer_abcsIterabletype)r   r;   kr}   r~   r   s   `   r   r~   z+Optimizer8bit.load_state_dict.<locals>.cast   s    %. **,1K!HHU[[1EE4(!KKM 2DAqD999#$44#5a#'q>a	2 E>#:#:;"tE{"A5"AAAr   ri   c                     | d   |d<   |S )Nr(   r   )r0   	new_groups     r   update_groupz3Optimizer8bit.load_state_dict.<locals>.update_group   s    "'/Ihr   )ri   r.   N)r   r.   rn   
ValueErroranyzipr   from_iterabler   r+   r   rh   )r   
state_dictgroupssaved_groups
param_lens
saved_lensold_idr2   id_mapri   r   r}   r   r   rq   ngr.   r~   s   `                @r   load_state_dictzOptimizer8bit.load_state_dict   s    j)
""!.1v;#l++#  87
==
N#j*2MNNC  !###F#FF###@#@@
 AI
 
	0 D!w'--/ 	DAqF{q	#E1~ea		
 .1-F
$)ArLB
 
 	E<HIa
Z
s   =E*Ec                    t        | j                        D ]  \  }}t        |d         D ]  \  }}|| j                  v s| j                  |   }|j                         D ][  \  }}t	        |t
        j                        s!t        |dd      }|r1|j                  |j                        | j                  |   |<   ]   y )Nr(   rX   F)
r,   r.   ri   r   r*   r5   r8   getattrr   r   )	r   gindexr0   pindexr2   valuesr   r}   rX   s	            r   to_gpuzOptimizer8bit.to_gpu   s    &t'8'89 	BMFE&uX7 B	

?!ZZ]F & B1%a6'.q*e'DH#+3444>

1a 0	BB	Br   c                 :   | j                   j                  D ]  \  }}}t        ||      }|J t        |t        j
                        st        |t        j                        sJ d}t        | j                        D ]  \  }}|r ot        |d         D ]  \  }}	|r "t        |	      t        |      k(  s"|| j                   j                  t        |	      <   | j                   j                  t        |	         | j                   j                  ||f<   d}   y )NFr(   T)r\   r    r   r*   r5   r8   r7   r,   r.   r-   r   r   )
r   r@   attrrB   pmodulefoundr   r0   r   r2   s
             r   check_overrideszOptimizer8bit.check_overrides   s   $(HH$H$H 	% FD&fd+G&&&gu||4
9   E!*4+<+<!= %!*5?!; 
%IFA!u7+ 6<++BqE2 !HH//16 --#V, !%
%%	%r   c                    d}|$t        j                         5   |       }ddd       g }| j                  s'| j                          | j	                          d| _        t        | j                        D ]  \  }}t        |d         D ]  \  }}|j                  | j                  |   }t        |      dk(  r| j                  ||||       | j                  |       | j                  ||||       t         j                  j                            | j                  rt         j                  j                          |S # 1 sw Y   xY w)zPerforms a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        NTr(   r   )r5   enable_gradrV   r   r   r,   r.   gradri   rn   
init_stateprefetch_stateupdate_stepcudasynchronizerX   )	r   closureloss	overflowsr   r0   r   r2   ri   s	            r   stepzOptimizer8bit.step   s*    ""$ !y! 	  "KKM#D 't'8'89 
	)MFE&uX7 	)	66>

1u:?OOE1ff=##A&  66:

&&(	)
	) == JJ""$ 9! !s   D>>Ec                    i }|d   |d<   |d   |d<   |d   |d<   |d   |d<   | j                   j                  |d<   | j                   j                  |d<   | j                   j                  |d<   | j                   j                  |d<   | j                   j
                  |d	<   | j                   j                  |d
<   ||f| j                  j                  v r*|j                  | j                  j                  ||f          |S )Nbetasepsweight_decaylrr`   min_8bit_sizepercentile_clipping
block_wise	max_unorm
skip_zeros)
argsr`   r   r   r   r   r   r\   r   r9   )r   r   r   r0   rB   s        r   
get_configzOptimizer8bit.get_config  s    .weu!&~!6~T{t#yy33|"&))"9"9(,		(E(E$%#yy33|"ii11{#yy33|Ftxx444MM$((//0@ABr   c                     t        d      )Nz(init_state method needs to be overriddenNotImplementedErrorr   r0   r2   r   r   s        r   r   zOptimizer8bit.init_state(  s    !"LMMr   c                     t        d      )Nz-The update_step method needs to be overriddenr   r   s        r   r   zOptimizer8bit.update_step+  s    !;
 	
r   c                 R   | j                   r|j                         dk  r"t        j                  |||j                        S t        j                  |j                  ||j                  d}t        j                  |d       | j                  j                  j                  |       |S )Ng     j@r   r   r   )rX   numelr5   
zeros_liker   rY   	get_pagedshapefillr[   paged_tensorsr?   )r   r2   r   buffs       r   get_state_bufferzOptimizer8bit.get_state_buffer0  st    }}	C##AU188DD ;;uQXXFDFF4OMM''..t4Kr   c                     | j                   rZ| j                  |   }|d   }t        |dd      }|r6t        j                  |d          d|v rt        j                  |d          y y y y )NrL   rX   FrM   )rX   ri   r   rY   prefetch_tensor)r   r2   ri   s1rX   s        r   r   zOptimizer8bit.prefetch_state:  sk    ==JJqMExBr:u5H!!%/2u$%%eHo6 % 	 r   )    Fr
   )r   r   r   r   r^   rh   r   r   r   r5   no_gradr   r   r   r   float32r   r   __classcell__ra   s   @r   rF   rF   ^   sf    4H$KJZ	B%. U]]_% %N"N

 ). 7r   rF   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z ej
                         d        Z ej
                         d        Z xZS )Optimizer2Statec                    d|k  st        d|       d|k  st        d|       t        |t              rW|j                  dd      j                  dd      j	                         j                  d      }|D cg c]  }t        |       }}t        t        |            D ]&  }d||   cxk  rdk  rn t        d	| d
||           d|k  st        d|       t        ||||      }t        | -  ||||       |=i }||d<   d|d<   |	|d<   |
|d<   ||d<   ||d<   ||d<   t        |      | _        || _        y || _        || _        y c c}w )N        Invalid learning rate: Invalid epsilon value: ( ),      ? Invalid beta parameter at index : Invalid weight_decay value: r   r   r   r   r`   d   r   r   r   r   r   )r   r*   strreplacestripsplitfloatrangern   r+   rU   r   r   r   optimizer_name)r   r   r(   r   r   r   r   r`   r   r   r   r   r   r   rX   bir_   ra   s                     r   r   zOptimizer2State.__init__F  s   " by6rd;<<cz6se<==eS!MM#r*223;AACII#NE',-!U1X-E-s5z" 	A%((S( 6qcE!H:F 	
 l".|n=  2U,O:x@<D!+D*-D&'$1D!*=D&'!+D )D!+D DI - DI,7 .s   ;Ec                    | j                  |||      }|d   dk(  rt        j                  }n*|d   dk(  rt        j                  }nt	        d|d          |j                         |d   k  rt        j                  }| j                  |   }d|d<   |t        j                  k(  s&|t        j                  k(  r]|j                         dk  rJ| j                  |t        j                  	      |d
<   | j                  |t        j                  	      |d<   nn|t        j                  k(  rZ|d   dk(  rd| j                  vr| j                          | j                  d   j                  |j                        | j                  d<   | j                  d   j                  |j                        | j                  d<   | j                  |t        j                  	      |d
<   | j                  d   |d<   | j                  |t        j                  	      |d<   | j                  d   |d<   |d   r|j                         }|dz  }	|	|dz  dkD  rdndz  }	t        j                  |	ft        j                  |j                        |d<   t        j                  |	ft        j                  |j                        |d<   nt        j                  dt        j                  |j                        |d<   t        j                  dt        j                  |j                        |d<   t        j                  dt        j                  |j                        |d<   t        j                  dt        j                  |j                        |d<   |d   dk  r$t        j                  d|j                        |d<   |d    d!kD  r%t        j                  d|j                        |d"<   y y )#Nr`   r   rT   (Amount of optimizer bits not supported: r   r   r      r   rL   rM   rd   re   rJ   rK   r         r   rN   rO   r   rH   rP   rI   rQ   r   r   r   r   rR   r   r   rS   r   r5   r   r   r   r   ri   r   rW   r^   r   r   zeros
r   r0   r2   r   r   rB   r   ri   nblockss
             r   r   zOptimizer2State.init_state{  s0   7,2%MMEL!Q&KKE%:6,;O:PQ  779vo..MME

1fEMM!U[[ QWWY%5"33AU]]3KE(O"33AU]]3KE(Oekk!V}!DNN2NN$,0NN9,E,H,HHH-y) .2^^J-G-J-JHH.z* #33AU[[3IE(O!^^I6E'N"33AU[[3IE(O!^^J7E'Nl#GGIdq4x!|!2#(;;IU]]188$i  $);;IU]]188$i  !&ahh!f %*KKahh%j! !&ahh!f %*KKahh%j! '(3.!&VAHH!EE++$!&T!((!CE+ %r   c                 J   | j                   |   }|j                  }| j                  |||      }|dxx   dz  cc<   |d   }|d   dk  r#t        j                  ||d   ||d         \  }	}
}nd}|d   j
                  t        j                  k(  r]t        j                  | j                  |||d   |d   d	   |d
   ||d   |d   |d   d   |d   ||d   dkD  r|d   nd |d   |d          y |d   j
                  t        j                  k(  r|d   st        j                  | j                  |||d   |d   |d   d	   |d   d   |d
   ||d   |d   |d   |d   |d   |d   |d   |d   ||d   dkD  r|d   nd |d          |d   |d   c|d<   |d<   |d   |d   c|d<   |d<   y |d   j
                  t        j                  k(  ra|d   r[t        j                  | j                  |||d   |d   |d   d	   |d   d   |d
   ||d   |d   |d   |d   |d   |d   ||d          y y y )Nr   r   r   r   rR   r   rL   r   r   r   r   rM   r   r   r   rS   r   r   r   r   rJ   rK   rH   rI   rP   rQ   )gnorm_scalerS   r   rN   rO   r   r   ri   r   r   rY   r   r   r5   r   optimizer_update_32bitr   r   optimizer_update_8bitoptimizer_update_8bit_blockwiser   r0   r2   r   r   ri   r   rB   r   current_gnorm
clip_valuer   s               r   r   zOptimizer2State.update_step  s   

1vv7fV}'(3.565J5JeK($7L0M62M:{ K?  EKK/$$##hw"uthw"~&&,[&9C&?k"T -!,/$ 8_""ekk1&:N####hhw"w"utggffj!j!~&'+&,  , --4 05Z/@%-,E&M5,/4Z/@%-,E&M5,8_""ekk1f\6J--##hhw"w"utggi i ~&'!,/# 7K1r   )MbP?)?g+?:0yE>r   r   Nr   r   Tr   FF	r   r   r   r   r5   r   r   r   r   r   s   @r   r   r   E  sn    
 3-j U]]_ED EDN U]]_S Sr   r   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z ej
                         d        Z ej
                         d        Z xZS )Optimizer1Statec                    d|k  st        d|       d|k  st        d|       t        t        |            D ]&  }d||   cxk  rdk  rn t        d| d||           d|k  st        d|       t        ||||      }t        |   ||||       |=i }||d	<   d
|d<   |	|d<   |
|d<   ||d<   ||d<   ||d<   t        |      | _        || _        y || _        || _        y )Nr   r   r   r   r   r   r   r   r`   r   r   r   r   r   r   )	r   r   rn   r+   rU   r   r   r   r   )r   r   r(   r   r   r   r   r`   r   r   r   r   r   r   rX   r   r_   ra   s                    r   r   zOptimizer1State.__init__  sA   " by6rd;<<cz6se<==s5z" 	A%((S( 6qcE!H:F 	
 l".|n=  2U,O:x@<D!+D*-D&'$1D!*=D&'!+D )D!+D DI - DI,r   c                 f   | j                  |||      }|d   dk(  rt        j                  }n*|d   dk(  rt        j                  }nt	        d|d          |j                         |d   k  rt        j                  }| j                  |   }d|d<   |t        j                  k(  s&|t        j                  k(  r9|j                         dk  r&| j                  |t        j                  	      |d
<   ni|t        j                  k(  rU|d   dk(  rSd| j                  vr| j                          | j                  d   j                  |j                        | j                  d<   | j                  |t        j                  	      |d
<   | j                  d   |d<   |d   rY|j                         }|dz  }	|	|dz  dkD  rdndz  }	t        j                  |	ft        j                  |j                        |d<   nft        j                  dt        j                  |j                        |d<   t        j                  dt        j                  |j                        |d<   |d   dk  r$t        j                  d|j                        |d<   |d   dkD  r%t        j                  d|j                        |d<   y y )Nr`   r   rT   r   r   r   r   r   r   rL   rd   rJ   r   r   r   r   rN   r   rH   rP   r   r   r   r   rR   r   r   rS   r   r   s
             r   r   zOptimizer1State.init_stateL  sL   7,2%MMEL!Q&KKE%:6,;O:PQ  779vo..MME

1fEMM!U[[ QWWY%5"33AU]]3KE(Oekk!V}!DNN2NN$,0NN9,E,H,HHH-y) #33AU[[3IE(O!^^I6E'Nl#GGIdq4x!|!2#(;;IU]]188$i  !&ahh!f %*KKahh%j! '(3.!&VAHH!EE++$!&T!((!CE+ %r   c                    | j                   |   }|j                  }| j                  |||      }|dxx   dz  cc<   |d   }|d   dk  r#t        j                  ||d   ||d         \  }	}
}nd}|d   j
                  t        j                  k(  rZt        j                  | j                  |||d   |d   d	   |d
   ||d   d |d   d   |d   ||d   dkD  r|d   nd |d   |d          y |d   j
                  t        j                  k(  r{|d   svt        j                  | j                  |||d   d |d   d	   |d   d   |d
   ||d   |d   d |d   d |d   d |d   ||d   dkD  r|d   nd |d          |d   |d   c|d<   |d<   y |d   j
                  t        j                  k(  rX|d   rRt        j                  | j                  |||d   d |d   d	   |d   d   |d
   ||d   |d   d |d   d |d   ||d          y y y )Nr   r   r   r   rR   r   rL   r   r   r   r   r   r   r   rS   r   r   r   rJ   rH   rP   )r   rN   r   r   r  s               r   r   zOptimizer1State.update_step  s   

1vv7fV}'(3.565J5JeK($7L0M62M:{ K?  EKK/$$##hw"utw"~&&,[&9C&?k"T -!,/$ 8_""ekk1&:N####hw"w"utgfj!~&&,[&9C&?k"T -). 05Z/@%-,E&M5,8_""ekk1f\6J--##hw"w"utgi ~&'!,/# 7K1r   )r  )r  r   r  r   r   Nr   r   Tr   FFr	  r   s   @r   r  r    sl    
 /-b U]]_5D 5Dn U]]_O Or   r  )collectionsr   r   r   copyr   	itertoolsr   r5   bitsandbytes.functional
functionalrY   r   r   optim	OptimizerrF   r   r  r   r   r   <module>r     sh   
 . #    #2 2FN FNRd7EKK)) d7NRm Rjzm zr   