o9i0PxddlmZmZmZmZddlZddlZddlmcm Z ddlm Z m Z m Z mZddlZddlZddlmZmZddlmZddlmZmZedd ZGd d ejj4ZGd d ejj4ZGddejj8ZGddej<ZGddeZ GddeZ!Gddejj8Z"dZ#Gddej<Z$Gddej<Z%Gddej<Z&y))OptionalTypeVarUnionoverloadN)Tensordevicedtypenn) undo_layout get_tile_inds)GlobalOptimManager) OutlierTracerfind_outlier_dimsTztorch.nn.Module)boundceZdZ ddededeedeedededed eed dffd Zdd Z dd Z ded efdZ xZ S)StableEmbeddingNnum_embeddings embedding_dim padding_idxmax_norm norm_typescale_grad_by_freqsparse_weightreturnc t |||||||||| | tjj || |_t jj|dddiyN)rweight optim_bits ) super__init__torchr LayerNormnormr get_instanceregister_module_override) selfrrrrrrrrrr __class__s a/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/bitsandbytes/nn/modules.pyr#zStableEmbedding.__init__sp            HH&&}V&D '')BB (\2. ctjjj|j|j yNr$r initxavier_uniform_r_fill_padding_idx_with_zeror)s r+reset_parametersz StableEmbedding.reset_parameters4(  %%dkk2 ((*r,c|jFtj5|j|jj ddddyy#1swYyxYwNrrr$no_gradrfill_r3s r+r2z+StableEmbedding._fill_padding_idx_with_zero?S    ' 7 D,,-33A6 7 7 ( 7 7 )AAinputc ^tj||j|j|j|j |j |j}|jtj}|j|j|jjSr.) F embeddingrrrrrrtor$get_default_dtyper&r r)r=embs r+forwardzStableEmbedding.forwardDs}kk  KK    MM NN  # # KK ffU,,./yy~  !2!233r,)NN@FFNNNrN) __name__ __module__ __qualname__intrfloatboolrr#r4r2rE __classcell__r*s@r+rrs &*$(#($(   c]  5/    !  &!   <+7 4V44r,rceZdZ ddededeedeedededed eed eed dffd Z dd Z ddZ ded efdZ xZ S) EmbeddingNrrrrrrrrrrc t ||||||||||  tjj |dddiyr)r"r#r r'r() r)rrrrrrrrrr*s r+r#zEmbedding.__init__VsW            '')BB (\2. r,ctjjj|j|j yr.r/r3s r+r4zEmbedding.reset_parametersqr5r,c|jFtj5|j|jj ddddyy#1swYyxYwr7r8r3s r+r2z%Embedding._fill_padding_idx_with_zero|r;r<r=c tj||j|j|j|j |j |j}|Sr.)r?r@rrrrrrrCs r+rEzEmbedding.forwardsHkk  KK    MM NN  # # KK  r,)NNrFFFNNrG)rHrIrJrKrrLrMrrr#r4r2rErNrOs@r+rQrQUs &*$(#($(#'   c]  5/    !  &!     6+7 V  r,rQc eZdZd dZdZeddedeee e fdeee e fde def dZeddedee e fde defd Zedded ede defd Zfd ZxZS) Params4bitc|tjd}tjj|||}||_||_||_||_||_|Sr7) r$emptyr_make_subclass blocksizecompress_statistics quant_type quant_statedata)clsr_ requires_gradr^r[r\r]r)s r+__new__zParams4bit.__new__sV <;;q>D||**3mD"#6 $&  r,c|jjjj|}tj j ||j|j|j\}}||_||_ |S)N)r[r\r]) r_ contiguoushalfcudabnb functional quantize_4bitr[r\r]r^)r)rww_4bitr^s r+rfzParams4bit.cudas II " ' ' ) . .v 6!nn::1lpmEmERVRaRa:b  & r,r)rr non_blockingrcyr.r)rr rls r+rAz Params4bit.to r,cyr.rnr)r rls r+rAz Params4bit.torpr,tensorcyr.rnr)rsrls r+rAz Params4bit.torpr,ctjjj|i|\}}}}|C|jdk(r4|j j jdk(r|j|S|j}||dj||d<|jrc|ddj||dd<|dddj||ddd<|dddj||ddd<tt |%||||j|j|j|j|j}|S)Nrfcpurrr rl)rar^r[r\r])r$_C_nn _parse_totyper_rrfr^rAr\rWr"rar[r]) r)argskwargsrr rlconvert_to_formats new_paramr*s r+rAz Params4bit.tosK9>9O9OQU9`Y_9`6|%6  6;;TYY=M=M=R=RV[=[99V$ $  A}twwv!++ !uQx{{62AbE!H"#B%(1+.."8AbE!HQK"#B%(1+.."8AbE!HQK"57:V5Wc:#d040B0BPTP`P`-1^^QUQiQi.2oo?I  r,)NTN@Tfp4....rHrIrJrbrfrrrrrKrr strrMrArrNrOs@r+rWrWs   HU3;%78 xPUV[]`V`PaGb x| IJ    5, D 1    F $   r,rWcFeZdZdfd ZdZdej fdZxZS) Linear4bitct|||||t|jjd|||_||_d|_y)NF)rar\r])r"r#rWrr_ compute_dtypecompute_type_is_set) r)input_featuresoutput_featuresbiasrr\r]rr*s r+r#zLinear4bit.__init__sK $G !1!1\o}GH *#( r,cd|jtjtjfvr|j|_y|jtj k(r|jtjk(rL|j |jdk(r,tjdtjdd|jtjk(rN|j |jdk7r-tjdtjddyyyy)NzInput type into Linear4bit is torch.float16, but bnb_4bit_compute_type=torch.float32 (default). This will lead to slow inference.ignorez .*inference.)messagezInput type into Linear4bit is torch.float16, but bnb_4bit_compute_type=torch.float32 (default). This will lead to slow inference or training speed.z.*inference or training) r r$float32bfloat16rfloat16numelshapewarningswarnfilterwarnings)r)xs r+set_compute_typezLinear4bit.set_compute_types 77u}}enn5 5"#D  WW %!!U]]2 QWWR[8P !bd''.I!!U]]2 QWWR[8P !tv'':ST9Q2&r,rc|ja|jj|jk7r>|jjj|j|j_t |j dd t d|js|j|d|_|j}|j|j|j}|jdn$|jj|j}tj||j j||j j}|j|}|S)Nr^zhFP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first.T)rr^)rr r_rAgetattrrprintrrrrg matmul_4bittr^)r)r inp_dtyperouts r+rEzLinear4bit.forwards 99 TYY__%?!YY^^..qww7DIIN 4;; t 4 < | }''  ! !! $'+D $GG    )T''(Ayy(tdiill4;M;M.NooatI`I`affY r,)TNTrN) rHrIrJr#rr$rrErNrOs@r+rrs) U*r,rc eZdZdfd ZxZS) LinearFP4c 0t||||||d|y)Nrr"r#r)rrrrr\rr*s r+r#zLinearFP4.__init__ $ Obdikqrr,TNTN)rHrIrJr#rNrOs@r+rrsssr,rc$eZdZdZdfd ZxZS) LinearNF4a7 Implements the NF4 data type. Constructs a quantization data type where each bin has equal area under a standard normal distribution N(0, 1) that is normalized into the range [-1, 1]. For more information read the paper: QLoRA: Efficient Finetuning of Quantized LLMs (https://arxiv.org/abs/2305.14314) Implementation of the NF4 data type in bitsandbytes can be found in the `create_normal_map` function in the `functional.py` file: https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L236. c 0t||||||d|y)Nnf4rrs r+r#zLinearNF4.__init__ rr,r)rHrIrJ__doc__r#rNrOs@r+rrs ssr,rc eZdZ d dZfdZe ddedeee e fdeee e fde def dZeddedee e fde defd Zedded ede defd Zfd ZxZS) Int8Paramsc||_d|_d|_|tjd}tj j |||Sr7)has_fp16_weightsCBSCBr$rYrrZ)r`r_rarrrs r+rbzInt8Params.__new__sE 0 <;;q>D||**3mDDr,c<|jrt| |S|jj j j|}t jj|\}}}}}~~||_t|d|t|d||S)Nrr) rr"rfr_rdrergrh double_quantsetattr) r)rBrCBtrSCBt coo_tensorBr*s r+rfzInt8Params.cuda"s  7<' ' $$&++-226:A.1nn.I.I!.L +BS$ DI D$ # D% % r,r)rr rlrcyr.rnros r+rAz Int8Params.to2s r,cyr.rnrrs r+rAz Int8Params.to;rpr,rscyr.rnrus r+rAz Int8Params.to?rpr,ctjjj|i|\}}}}|C|jdk(r4|j j jdk(r|j|Stt|)||||j|j}|j|_ |j|_|S)Nrfrwrz)rar)r$r{r|r}r~r_rrfrr"rArarrr) r)rrrr rlrrr*s r+rAz Int8Params.toCs9>9O9O : : 6|%6   v%   %%.99V$ $" !\#00!%!6!6 I 77IL HHIM r,)NTFNNrrrrOs@r+rrs   E 03-0   sF{+, eSj)*       5, D 1    F $   r,rc|j|d}|y|j|dd}|dk7r)t||j} t || ||d<yy)Nr weight_formatrow)getpopr rr ) state_dictprefixlocal_metadatastrict missing_keysunexpected_keys error_msgsrr tile_indicess r+maybe_rearrange_weightr\sk ^^vhf- .F ~NNfX]#;UCM$]FMMB (3FL(I fXV$%r,c^eZdZ dfd ZfdZfdZdZdejfdZ xZ S) Linear8bitLtc t ||||||rJdtj|_||_||j_||j_||j_|dkDr|sd|j_ t|jj|||_ |jty)Nzvmemory_efficient_backward is no longer required and the argument is deprecated in 0.37.0 and will be removed in 0.39.0Trra)r"r#rg MatmulLtStatestateindex thresholdrmemory_efficient_backwarduse_poolrrr_"_register_load_state_dict_pre_hookr r)rrrrrrrrr*s r+r#zLinear8bitLt.__init__is $G, g/g g,&&(  ( &6 #/H , s?#3"&DJJ  !1!1DTdtu  //0FGr,ct ||||d}t|j|}t|j|}|jj du}||z}|dz} |jj st||r|n|j||<d|| <y||s|r|n|j||<d|| <y|1|r|n|j||<|jj|| <yyy)Nrrr) r"_save_to_state_dictrrrCxBrdetachformatB) r) destinationr keep_varsscb_nameparam_from_weightparam_from_statelayout_reorderedkey_name format_namer*s r+rz Linear8bitLt._save_to_state_dictys #KC$DKK:"4::x8::>>5xj). zz** ,=F(9L]LdLdLf H%+0 K(!-6F>-%)[[__DJJN&&s+ ,r,c|jj|j_|jj|j_d|j_d|j_yr.rrrrr3s r+init_8bit_statezLinear8bitLt.init_8bit_state>     r,rc|j|j_|jj|j |j a|j j|jk7r>|j jj|j|j _tj||j|j |j}|jjs]|jjG|jj1|j`|jj|j_|SN)rr)trainingr is_trainingrrrrr r_rArgmatmulrrr)rrs r+rEzLinear8bitLt.forwards!%  ;;>> %  " 99 TYY__%?!YY^^..qww7DIINjjDKKdiitzzJzz**zz}}(TZZ^^-GJJM#'::>>   r,TTFrNN) rHrIrJr#rrrr$rrErNrOs@r+rrhs.TXZ^H >6,, r,rc2eZdZdfd ZdZdZdZxZS)OutlierAwareLinearcFt|||||d|_d|_y)NF)r"r# outlier_dim is_quantized)r)rrrrr*s r+r#zOutlierAwareLinear.__init__s& $G!r,ctd)NzJPlease override the `forward_with_outliers(self, x, outlier_idx)` functionNotImplementedError)r)r outlier_idxs r+forward_with_outliersz(OutlierAwareLinear.forward_with_outlierss!"noor,ctd)NzEPlease override the `quantize_weights(self, w, outlier_idx)` functionr)r)rjr s r+quantize_weightz"OutlierAwareLinear.quantize_weights!"ijjr,c||jQtj}|js t d|j |j }||_|jsS|j|j |j}|j jj|d|_yy)NzTPlease use OutlierTracer.initialize(model) before using the OutlierAwareLinear layerT) rrr'is_initializedr get_outliersrrr r_r)r)rtracerr rjs r+rEzOutlierAwareLinear.forwards    #"//1F((*lm --dkk:K*D   $$T[[$2B2BCA KK   " "1 % $D !r,)TN)rHrIrJr#r r rErNrOs@r+rrs" pk %r,rc8eZdZ dfd ZdZdZxZS)SwitchBackLinearBnbc Nt |||||tj|_||_||j_||j_||j_|dkDr|sd|j_ t|jj|||_ y)NrTr) r"r#rgrrrrrrrrrr_rs r+r#zSwitchBackLinearBnb.__init__s  OT6 &&(  ( &6 #/H , s?#3"&DJJ  KK  /?O_  r,c|jj|j_|jj|j_d|j_d|j_yr.rr3s r+rz#SwitchBackLinearBnb.init_8bit_staterr,c2|j|j_|jj|j t j|j|jjd|j|jz}yr) rrrrrrrg matmul_mixedrerrs r+rEzSwitchBackLinearBnb.forwardse!%  ;;>> %  "qvvx)9)9);$djjY\`\e\eer,r)rHrIrJr#rrErNrOs@r+rrs' "' 6 fr,r)'typingrrrrrr$torch.nn.functionalr rhr?rrr bitsandbytesrgbitsandbytes.functional bitsandbytes.autograd._functionsr r bitsandbytes.optimr bitsandbytes.utilsrrrrQr ParameterrWLinearrrrrrrrrrnr,r+r!s 65 ++G1? C()=4ehh((=4@7""7r<##<|00ds s s s G##GT JY299Yx%%4(f"))(fr,