m9i6+ ddlZddlZddlZddlZddlZddlZddlZddlmZddl Z ddl m Z ddl mZddlmZddlmZmZdZiZer iZej,ej.ej0fed <ej2ej4fed <ej6ej8fed <ej:ej<ej>fed <ej@ejBfed <iZ"ejFejHfe"d <ejJejLfe"d <ejNejPfe"d <ejRejTfe"d <ejFejHfe"d<ejJejLfe"d<iZ+ejXejZej\fe+d <ej^ej`fe+d <ejbejdfe+d <ejfejhejjfe+d <ejlejnfe+d <GddZ8GddZ9GddZ:iZ;de;ejx<de;ejz<de;ej|<de;ej~<de;ej<ejxejddddZBddZCddZDddZEddZFdd ZGdd!ZHdd"ZIdd#ZJdd$ZKdd%ZLd&ZMd'ZNd(ed)ejfd*ZPd+ZQd,ZRdd-ZS dd.ZT dd/ZUdd(ed0ed1eVd)efd2ZWdd(ed3ed4ed0ed)ef d5ZX dd(ed6eeefd4ed3ed0ed7eYd)efd8ZZdd9Z[dd(ed4ed0efd:Z\dd(ed4ed0efd;Z]dd(ed4ed0ed)efd<Z^dd(ed6eeefd4ed0ed7eYd)ef d=Z_dd(ed6eeefd4ed0ed7eYd)ef d>Z`dd(ed6eeefd4ed0ed7eYd)ef d?Zadd(ed3ed0ed)efd@Zb dd(ed6eeefd4ed3ed0ed)ef dAZcdd(ed3ed0ed)efdBZddd(ed3ed0ed)efdCZe ddDefdEedFedGedHeVdIeVdJeYdKeVdLedMeVdNeVdOeVdPedQeVd)dfdRZg ddDefdEedFedGedLedHeVdMeVdIeVdJeYdKeVdSedTedUedVedWedXedNeVdOeVdPedQeVd)df*dYZh ddDefdEedFedGedLedHeVdMeVdIeVdJeYdKeVdSedTedZed[edNeVdOeVd)df"d\Zi dd]ed^edJeYd_eYfd`ZjdaedbedceddefdeZkejfdfZl dd(edged0efdhZm dd(edged0efdiZn dd(edged0efdjZoddejfdkZq ddlZr ddmZsGdndoZtGdpdqZuGdrdsZvdtZwduZxejfdvZz ddwZ{ddxZ|ddyZ}ddzZ~d{Zdd}Zdd~Zejd|fdZejfdZdZdZy)N)norm)reduce)Tuple)Tensor)COMPILED_WITH_CUDAlibc8ttj|dSNr)roperatormul)iterables a/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/bitsandbytes/functional.pyprodrs (,,! ,,adammomentumrmsproplionadagradlamblarsc4eZdZdZdZdZedZddZy)GlobalPageManagerNctdNzCall get_instance() instead RuntimeErrorselfs r__init__zGlobalPageManager.__init__c899rcg|_yN) paged_tensorsrs r initializezGlobalPageManager.initializefs rc|j0|j||_|jj|jSr$ _instance__new__r&clss r get_instancezGlobalPageManager.get_instancei6 == KK,CM MM $ $ &}}rcJ|jdddD]}t||y)N)r%prefetch_tensor)r to_cputs r prefetch_allzGlobalPageManager.prefetch_allps,##DbD) 'A Av & 'rF) __name__ __module__ __qualname__r)r!r& classmethodr-r4rrrr`s*I:  'rrc2eZdZdZdZdZedZdZy)CUBLAS_ContextNctdrrrs rr!zCUBLAS_Context.__init__|r"rci|_yr$)contextrs rr&zCUBLAS_Context.initializes  rc|j0|j||_|jj|jSr$r(r+s rr-zCUBLAS_Context.get_instancer.rc|j|jvrtjj }tjj |t jtj|j|j<tjj ||j|jSr$) indexr?torchcudacurrent_device set_devicectc_void_pr get_context)r device prev_devices rrIzCUBLAS_Context.get_contexts} <>>: ??AGG$QWWY.IMM'!*bkk)4bjj6JKrc d}|jtjk(r+ttd|dd}t j |}nG|jtjk(r*ttd|dd}t j|}|td|t|dd}|r|rt|| t||t|t|t j|j|js |jrtjj!yy)Nc_fp32_uint8zFunction not implemented: is_managedF)rUrCfloat32getattrr rGc_floatuint8c_uint8NotImplementedErrorr1roc_int64rmrdrD synchronize) func_namerqBvalueprefetchfunccvaluerws relementwise_funcrs Dww%--sa {%0$7E" EKK sa {&148E" |.1KI;/WXXL%0Jh =/!,WQZAGGI)>?zzQZZ    rc td|d|y)Nfillr)rqrrJrs rrrs0@DRW0Xrc td|ddy)Narangerr)rqrJs rrrs,Xq$Brc td||dy)N_mulrr)rqrrJs rrrs-faA>rcN|rdnd}d|z}|s|dkr|sd|znd|zdz }tj|d|}d|jz }|dk(r|S|jdz}tj|d|j dg|zz||dj zS) NrSr?r)rClinspacermrtolist)signed total_bitsadd_zerosign total_valuesvaluesgapls rcreate_linear_maprsDDj=L:> .4: Jq ^^D#| 4F   C ax LLNA ||F2AJ--/1#c'9F12JS  MrcT|}|}|rdnd}||z||z k(sJg}g}ttd||z z d||z zdD]\} } |jd| zg} tt j ddg|} d|dz z} td|zD]}| D]z}|dk7rdnd}tt|D]\} }||d| dz zzz }|dk(r |d| zz}n|d|| z dz zz}| j||si| j| |t | d|zk(sJ| j|dkr/dt | z }t|D]} | jd| jtj| }||jz}|S)NrrrS)repeatrr) enumeraterangeappendlist itertoolsproductlenrrCrr)r exponent_bitsprecision_bitsrephas_signevaluespvaluesivalrlstbiasevalue bit_patternrpvalrcodes rcreate_fp8_maprsAAqH Q3*X% %% %GGEQx)?%@"BA V^H^D_abcd3q#vF y  !Q? @C }Q DM*+ & &K A+Q1E$T+%67 *4qAaC&y)) *{a4j(a6$;q=!111 MM%  uf% & & v;!Z- '' ' KKMA~CKs A MM!   KKM << DDHHJD Krc@g}||rdndz }d||z zdz }|sd|z}t|D]}t|rd||z|z zdznd||z|z dzzdz}tjdd|}|dd|ddzdz } |d|dz |zz| zj z }|r!|d|dz |zz | zj z }|dkDstjdd|dz}|dd|ddzdz } |d|dz |zz| zj z }|s|d|dz |zz | zj z } |j d|j d d t |z } t| D]}|j d|jt|S) a+ Creates the dynamic quantiztion map. The dynamic data type is made up of a dynamic exponent and fraction. As the exponent increase from 0 to -7 the number of bits available for the fraction shrinks. This is a generalization of the dynamic type where a certain number of the bits and be reserved for the linear quantization region (the fraction). n determines the maximum number of exponent bits. For more details see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561] rrrSg?Nr0@ rr) rintrCrrrrrr) rmax_exponent_bitsrdata non_sign_bitsadditional_itemsrfraction_items boundariesmeansrs rcreate_dynamic_mapr0s0" Dv!15M]->>?!C // $ % SRXaA $58I$IJQN^_dehuduyJeJMNeN_ORS_SU^^CN; CR:ab>1S8 ",q01A56%?GGII  r 1A 56:;?@5HPPRR S KKNKK D /C 3Z A IIK $<rcDt|d|zdz }|j}|jddt|z }t |D]}|jd|j t |}||jjz }|S)NrSr) num_quantilesrr) estimate_quantilesrrrrrrabsr)rqrqrrs rcreate_quantile_maprbs1AzM!O+s? Frjrc :|jdkrtd|jd|dkDrtd||dkr |dk(rdd|zz }|0tjdtj|j }t ||gt|j }|jtjk(r_tjt|t|tj|tj|jn|jtjk(r_tj t|t|tj|tj|jntd |jt#||dkrQt%d|z }tj&d d |j)j+|j }||}|S) a Estimates 256 equidistant quantiles on the input tensor eCDF. Uses SRAM-Quantiles algorithm to quickly estimate 256 equidistant quantiles via the eCDF of the input tensor `A`. This is a fast but approximate algorithm and the extreme quantiles close to 0 and 1 have high variance / large estimation errors. These large errors can be avoided by using the offset variable which trims the distribution. The default offset value of 1/512 ensures minimum entropy encoding -- it trims 1/512 = 0.2% from each side of the distrivution. An offset value of 0.01 to 0.02 usually has a much lower error but is not a minimum entropy encoding. Given an offset of 0.02 equidistance points in the range [0.02, 0.98] are used for the quantiles. Parameters ---------- A : torch.Tensor The input tensor. Any shape. out : torch.Tensor Tensor with the 256 estimated quantiles. offset : float The offset for the first and last quantile from 0 and 1. Default: 1/(2*num_quantiles) num_quantiles : int The number of equally spaced quantiles. Returns ------- torch.Tensor: The 256 quantiles in float32 datatype. rzQQuantile estimation needs at least 256 values in the Tensor, but Tensor had only z values.zgCurrently only a maximum of 256 equally spaced quantiles are supported, but the argument num_quantiles=`?rrS)rrTzNot supported data type r)rmr}rCrrxrJrrrUr cestimate_quantiles_fp32rorGrzr^float16cestimate_quantiles_fp16rroundrlongto)rqrjrrrJstepidxs rrrs: wwy315FGHGNGNGPFQQY3Z[[s"59`an`o7p#qqsv0AmO$ {%++fEMM!((SC q#h ahh Fww%-- $$WQZrzz&?QSUS[S[\]\c\c\eSfg EMM ! $$WQZrzz&?QSUS[S[\]\c\c\eSfg!$t || }||z }|jtj k7r|j}|,t j|j||j}|jjdk7r1t|j} |j|j}|dvrtd|dt|||g|jtj k(rttj t#|t#|t#|t#|t%j&|t%j&|j)n8|jtj*k(rstj,t#|t#|t#|t#|t%j&|t%j&|j)n|jtj.k(rstj0t#|t#|t#|t#|t%j&|t%j&|j)ntd |jt3|j|S|j5}tj6t#|d t#|t#|d t#|t%j8|t%j8|j)|S) aQ Dequantizes blockwise quantized values. Dequantizes the tensor A with maximum absolute values absmax in blocks of size 4096. Parameters ---------- A : torch.Tensor The input 8-bit tensor. quant_state : tuple(torch.Tensor, torch.Tensor) Tuple of code and absmax values. absmax : torch.Tensor The absmax values. code : torch.Tensor The quantization map. out : torch.Tensor Dequantized output tensor (default: float32) Returns ------- torch.Tensor: Dequantized tensor (default: float32) Nr%FrTr(r+r*r,r-rr.r/The blockwise of J is not supported. Supported values: [2048, 4096, 1024, 512, 256, 128, 64]r0rr)r3rr rJrCrxdequantize_blockwiserUfloatemptyrWrrrrr cdequantize_blockwise_fp32rorGr^rmrcdequantize_blockwise_fp16r7cdequantize_blockwise_bf16rr(cdequantize_blockwise_cpu_fp32r:) rqrAr#rrjr1r2rUrr@rJs rrFrFsD  "f&88 8 | + I %#5#7#:#:188#DIi #dIuemmT4P{=H:FD)VUFF %ff5& <<5== (6<<>& {kk!''qxx@xx}}!((#wwqxx  A A0 +BGAJPWXcdeXfPgipqtiuwyxExEFOxPRTR_R_`a`g`g`iRj k JrcR|d}d}|dk(r gd}n4|dk(rgd}n*|dk(rgd}n |dk(r|d k(r gd ddd }n td |td |dt|}||jjz}|j dk(sJ|j |S)NrDnf4)rg 6GgfgTFٿgI4ҿg০ǿg Org__?g`\?g?g@g?g4?g`?g`v"?rfp4)rg?g @g(@g@g@rg@rgg g(ggggint4)rrRrrSrrrr0iiaf4r/)rg|8geg:Kڞ׿gH2퓊cпg}Yu-ÿgQ #(DrgF?g`_?g 0E?gL_߹E?gƶ=?ga@?gкv -?rr0z94-bit AbnormalFloats currently only support blocksize 64.z Typename z not supported)r}rrrrmr )typenamerJr1rs r get_4bit_typerZs ~v D5 ) U l V G U  ???CtED&(ac c |!IhZ~"FGG $& {kk%uQXX> Aahh F q&# yyEMM!    . .wt}gaj'RX/[bcf[gikiqiqr{i|AGGHIJ K  . .wt}gaj'RX/[bcf[gikiqiqr{i|AGGHIJ K emm #    . .wt}gaj'RX/[bcf[gikiqiqr{i|AGGHIJ K  . .wt}gaj'RX/[bcf[gikiqiqr{i|AGGHIJ K enn $    . .wt}gaj'RX/[bcf[gikiqiqr{i|AGGHIJ K  . .wt}gaj'RX/[bcf[gikiqiqr{i|AGGHIJ KZ[\[b[bZcdee ahhWWQZ1_T%MSUUWn*rc|Vdtvr*tj|jtd<td}|j|j}t j |j }|jtjk7r|j}||z }t|||}|||ffS)Nr%) r3rr rJrCrrrUrxrGquantize_no_absmax)rqrrjr#inps rquantizers | I %#5#7#:#:188#DIi #wwqxx  YYq\   F ||u}}$v||~f f*C S$ ,C  rc||J|X|Vdtvr*tj|jtd<td}|j|j}|||f}t ||d|}||dzS)Nr%rr)r3rr rJdequantize_no_absmax)rqrAr#rrjs r dequantizers  "f&88 8 | + I %#5#7#:#:188#DIi #wwqxx tn q+a.# 6C Q rc Vt|j}|%tj|tj}t ||gt jt|t|t|tj|jt||S)a Quantizes input tensor to 8-bit. Quantizes the 32-bit input tensor `A` to the 8-bit output tensor `out` using the quantization map `code`. Parameters ---------- A : torch.Tensor The input tensor. code : torch.Tensor The quantization map. out : torch.Tensor, optional The output tensor. Needs to be of type byte. Returns ------- torch.Tensor: Quantized 8-bit tensor. r') rrJrCr4r{rr cquantizerorGr^rmrrqrrjrKs rrrsr*188$K {%**1EKK@C q#hMM'$-WS\288AGGI;NO k Jrc Xt|j}|%tj|tj}t |||gt jt|t|t|tj|jt||S)a Dequantizes the 8-bit tensor to 32-bit. Dequantizes the 8-bit tensor `A` to the 32-bit tensor `out` via the quantization map `code`. Parameters ---------- A : torch.Tensor The 8-bit input tensor. code : torch.Tensor The quantization map. out : torch.Tensor The 32-bit output tensor. Returns ------- torch.Tensor: 32-bit output tensor. r') rrJrCr4rxrr cdequantizerorGr^rmrrs rrrst*188$K {%**1EMMBC tQnOOGDM71:ws|RXXaggi=PQ k Jroptimizer_namegrstate1beta1epsr!lrr@beta2 weight_decay gnorm_scale unorm_vec max_unormcd}| dkDr-tj|jj}d}|jtj k(r t |d}n|jtjk(r t |d}nd|jtjk(r"tt |dk(r t |d}n%td|jd|jt||||| gt|j}|t|t|t|t|t| tj | tj |tj |tj | tj |tj | tj"|tj |tj | tj$|tj"|j't)|y) a$ Performs an inplace optimizer update with one or two optimizer states. Universal optimizer update for 32-bit state and 32/16-bit gradients/weights. Parameters ---------- optimizer_name : str The name of the optimizer: {adam}. g : torch.Tensor Gradient tensor. p : torch.Tensor Parameter tensor. state1 : torch.Tensor Optimizer state 1. beta1 : float Optimizer beta1. eps : float Optimizer epsilon. weight_decay : float Weight decay. step : int Current optimizer step. lr : float The learning rate. state2 : torch.Tensor Optimizer state 2. beta2 : float Optimizer beta2. gnorm_scale : float The factor to rescale the gradient to the max clip value. unorm_vec : torch.Tensor The tensor for the update norm. max_unorm : float The maximum update norm relative to the weight norm. skip_zeros : bool Whether to skip zero-valued gradients or not (default: False). rNrrrrSAGradient+optimizer bit data type combination not supported: grad , optimizer )rCrrrGrUrxstr2optimizer32bitrr7rrrrrJrorGrzrpc_boolrmr)rrrrrrr!rr@rrrrr skip_zeros param_norm optim_funcrKs roptimizer_update_32bitrspJ3ZZ / Jww%--'7: EMM !'7: ''U^^ #,>~,N(OQR(R'7: \]^]d]d\eeqrxr~r~qAB B q!VVY/0188$K    9 : 5 5 3 <  4 2 ; * 1779!"krqmap1qmap2max1max2new_max1new_max2chd}|dkDr-tj|jj}t |j }t |||||| | | | ||g |jtjk(rt|jtjk(rVt|dt|t|t|t|t|tj|tj|tj|tj|tj|tj|tj| t| t| t| t| t|t|tj|tj|tj|jn|jtj k(rs|jtjk(rUt|dt|t|t|t|t|tj|tj|tj|tj|tj|tj|tj| t| t| t| t| t|t|tj|tj|tj|jn%t#d|jd|jt%|y)a Performs an inplace Adam update. Universal Adam update for 32/8-bit state and 32/16-bit gradients/weights. Uses AdamW formulation if weight decay > 0.0. Parameters ---------- optimizer_name : str The name of the optimizer. Choices {adam, momentum} g : torch.Tensor Gradient tensor. p : torch.Tensor Parameter tensor. state1 : torch.Tensor Adam state 1. state2 : torch.Tensor Adam state 2. beta1 : float Adam beta1. beta2 : float Adam beta2. eps : float Adam epsilon. weight_decay : float Weight decay. step : int Current optimizer step. lr : float The learning rate. qmap1 : torch.Tensor Quantization map for first Adam state. qmap2 : torch.Tensor Quantization map for second Adam state. max1 : torch.Tensor Max value for first Adam state update. max2 : torch.Tensor Max value for second Adam state update. new_max1 : torch.Tensor Max value for the next Adam update of the first state. new_max2 : torch.Tensor Max value for the next Adam update of the second state. gnorm_scale : float The factor to rescale the gradient to the max clip value. unorm_vec : torch.Tensor The tensor for the update norm. max_unorm : float The maximum update norm relative to the weight norm. rrrrrN)rCrrrGrrJrrUrxr{str2optimizer8bitrorGrzrprmrrr)rrrrr@rrrr!rrrrrrrrrrrrrKs roptimizer_update_8bitrdsPJ3ZZ / 188$K q!VVYudD(T\]^ww%--FLLEKK$?.)!, AJ AJ FO FO I  JJy ! JJz " JJu  JJu  JJsO JJt  JJrN EN EN DM DM H  H  JJ| $ JJ{ # JJqwwy !+ . EMM !fllekk&A.)!, AJ AJ FO FO I  JJy ! JJz " JJu  JJu  JJsO JJt  JJrN EN EN DM DM H  H  JJ| $ JJ{ # JJqwwy !+ 0OPQPWPWyXdekeqeqdr s  krabsmax1absmax2c$d}t|j}t||||| | | | g|jtj k(r*|jtj k(r t|d}n|jtjk(r*|jtj k(r t|d}n|jtjk(r?|jtj k(r"tt|dk(r t|d}n%td|jd|jt|t||||| | | | gt|j}|t|t|t|t|tj|tj|tj|tj |tj| t| t| t| t| tj|tj|tj"|tj |j%t|y)NrrrrSrr)rrJrrUrCrxr{str2optimizer8bit_blockwiserr7rrrrorGrzrprrm)rrrrr@rrrr!rrrrrrrrrrKs roptimizer_update_8bit_blockwisers(J188$K q!VVUE7GDEww%--FLLEKK$?0@C EMM !fllekk&A0@C ''U^^ #  (C ).9 :A =0@C OPQPWPWyXdekeqeqdr s  k q!VVUE7GDE188$K   5 5 3 4 2 <  ; * 1779#&krgrad gnorm_vec percentilec 6t|j}t||g|jtj k(r_t jt|t|tj|tj|jn|jtjk(r_t jt|t|tj|tj|jntd|jdt|t j ||dz}t j"|\}}t j ||}d} ||kDr||z } ||| fS)aApplies percentile clipping grad: torch.Tensor The gradient tensor. gnorm_vec: torch.Tensor Vector of gradient norms. 100 elements expected. step: int The current optimiation steps (number of past gradient norms). zGradient type z not supported!dr)rrJrrUrCrxr cpercentile_clipping_g32rorGrprmrcpercentile_clipping_g16rrsqrtr) rrr!rrK current_gnormvalsr" clip_valuers rpercentile_clippingr%s94;;'K tY  zzU]]" $$ DM I  JJt  JJtzz| $  u}} $ $$ DM I  JJt  JJtzz| $  >$**_EFF kJJy45M 9%ID#D,-JKz! =0 *k 11r histogramindex1index2sourcec,t|jdk(sJ|jtjk(sJ|jtjk(sJ|jtj k(sJ|jtj k(sJ|j jdk(sJ|j jdk(sJ|j jdk(sJ|j jdk(sJtj|jd}tj|j}t||||gtjt|t|t|t|||y)NrSrDr)rrWrUrCrxint32rJrrGrprmrr chistogram_scatter_add_2dro)rrrrmaxdim1rs rhistogram_scatter_add_2drQs? y 1 $$ $ ??emm ++ + <<5== (( ( <<5;; && & <<5;; && &    F ** * ==   '' ' ==   '' ' ==   '' 'jj+,G 6<<>"A y&&&12!!')"4gfowvX_`fXgiprstrc tjjstjj|j|k7s|j|k7r%t d|jd|j|j }|j }|}|} d} t|dk(rt|dk(r|s%| s#|j d|j dk7rd} n|r%| s#|j d|j dk7rd} n|r%| r#|j d|j dk7rd} n|s| r|j d|j dk7rld} nht|dk(rt|dk(r|s%| s#|j d|j dk7rd} n%|r$| s"|j d|j dk7rd} n|r$| r"|j d|j dk7rd} n|s| r|j d|j dk7rd} nt|dk(rt|dk(r|s$| s"|j d|j dk7rd} nq|r$| s"|j d|j dk7rd} nK|r$| r"|j d|j dk7rd} n%|s#| r!|j d|j dk7rd} |a|j } | syt|dk(rjt|dk(r[| d|dk(rO| d|dk(rC|d|dk(r7|d|dk(r+d} n't|dk(rJt|dk(r<|s | s |d|df} n|r | r |d|df} n|r | s |d|df} n|s| r|d|df} nt|dk(rZt|dk(rL|s| s|d|d|df} n|r| r|d|d|df} n|r| s|d|d|df} nz|sx| rv|d|d|df} ngt|dk(rYt|dk(rK|s| s|d|d|df} n8|r| r|d|d|df} n%|r| s|d|d|df} n|s| r|d|d|df} | std |d |d |d | d  S) Nz3Expected torch.int8 input tensors A and B, but got  and TrSrrFrz?Tensor dimensions incorrect for matrix mulitiplication: A x B:  x z with transpose for A x B: .) rCrDis_initializedinitrUrrWrr) rqrrj transposed_A transposed_B expected_typesAsBtAtBcorrectsouts r check_matmulrdsu :: $ $ & (9ww-177m#;A!''%PQPWPWy Y   B B B BG 2w!|B1 "qwwqz!9G qwwqzQWWQZ7G B1771:3GqwwqzQWWQZ7G RA#b'Q,"qwwqz!9G qwwqzQWWQZ7G B1771:3GqwwqzQWWQZ7G RA#b'Q,"qwwqz!9G qwwqzQWWQZ7G B1771:3GqwwqzQWWQZ7G yy3r7aio supported for tensor contractions, but dims for A x B were: r)rrCrrrJrrW batched_igemmstriderr<r-rIrr cigemmrGrrpro)rqrrjrrrrrrrrrrrrs rigemmrs 1c< >D {kkt5;;qxxH 177|qS\Q. 771: # aggaj(@ As+ + B BB1 eRU^ #b'Q,eRUBqE "B1 eRU^ #b'Q,eRUBqE " 2w!| 88:a=AGGAJ & L XXZ]aggaj (L qww<1 xxz!} *$ A!''!*,# xxz!} *$ A!''!*,# r7a<1A((*,QA6C W\c"gl11 AQ%C qE qEhhj|!4e RA2w!||1A2a5BqE>_`b_ccfgifjk   qE qE qEBqEMe  % % ' 3 3AHH =Cq!SkJJsBIIl+RYY|-DbjjQRmUWU_U_`aUbdfdndnopdqqz71:ws|RZZ_bjjQToWYWaWabeWfh Jrct|jdk(rt|jdk(s%td|jd|jt|||||}|0t j |tj |j}|jr|jd}d}n|j}|d|jdk7r$|j}|jd}n|d|jdk(rd }|jd}n{|ddk(r$|j}|jd}nO|ddk(r$|j}|jd}n#|j}|jd}|jr|jd}d}n|j}|d|jdk7r&|j}|jd}d}nP|d|jdk(r|jd}d }n%|j}|jd}d}|jd} |jd} |jd} |jd} | } |jd|jdz}|jd|jdz}|jd|jdz}tjj|j}t|||gtj |t#j$|t#j$|t#j&| t#j&| t#j&| t)|t)|t)|t#j&|t#j&|t#j&| t#j*|t#j*|t#j*|t#j,| |S) Nrz@Expected 3-dimensional tensors for bmm, but got shapes A and B: rrrFrrST)rrWrrrCrrrJ is_contiguousr contiguousr<r-rIrr cbatched_igemmrGrrproc_longc_uint32)rqrrjrrrrsr num_batchrrrrstrideAstrideBstrideCrs rrrSsv qww<1 CLA$5NqwwiW\]^]d]d\e f   1c< >D {kkt5;;qxxHhhjm HHJ Q41771:  A((*Q-C qTQWWQZ L((*Q-CtqyLLNhhjm1LLNhhjmLLNhhjmhhjm HHJ Q41771:  A((*Q-C L qTQWWQZ ((*Q-CL A((*Q-C L I  A  A  A Cggaj1771:%Gggaj1771:%Gggaj1771:%G  % % ' 3 3AHH =C q!SksBIIl3RYY|5LbjjYZm]_]g]ghi]jlnlvlvwxlyqz71:ws|RZZ_bjjQToWYWaWabeWfyy!299W#5ryy7I2;;W`Kac Jrc |d}|d}t|} t|} | dk(sJd| dk(r|d} n| dk(r |d|dz} |dx} } tt|dkDs Jd||ddk(r:| dk(r5tjd|df|j tj S|ddk(rH| dk(rCtjt|dd|dgz|j tj S| dk(r'|%t|d|df||j dd \}}n/| dk(r*|(t|d|d|df||j dd \}}| dk7sJd |j jd k(sJ|j jd k(sJ|jtjk(sJ|jtjk(sJ|j|k(sJ|ddk(sJ|dd vsJ|ddk(sJ|d |d k(s Jd|d||d}|j }tjj|j tjj!|j }t#|}t#|}t#|}|d }t%j& dz}|dk(r"t%j&| dzdzdzdz}n!t%j&| dzdzdzdz}t%j&| dz}t%j&| } t%j&| } t%j&|}d}t#d}t)|||g|dk(rS|tj*k(r t-j.|| | |||||||| }nwt-j0|| | |||||||| }nW|dk(rR|tj*k(r t-j2|| | |||||||| }nt-j4|| | |||||||| }|dk(r.t7d|d|d|dd|||fd| | |f t9dtjj|||fS)NrrSz:Only two dimensional matrices are supported for argument Brrz(Input tensor dimensions need to be > 0: r&rrzlen(B.shape)==3 not supportedrDrrr0zNMatmullt only supports A @ B^T. Inner matrix dimensions do not match: A @ B = z @ rrrrrrzA: z, B: z, C: z; (lda, ldb, ldc): z ; (m, n, k): zcublasLt ran into an error!)rrrrCrHrJrtuplerrrUrrDrFr<r-rIrorGrprrr cigemmlt_turing_32cigemmlt_turing_8cigemmlt_ampere_32cigemmlt_ampere_8r Exception)rqrSASBrjSoutrUshapeAshapeBdimsAdimsBrr rformatBrKrptrAptrBptrCrrrr has_error ptrRowScales rigemmltr s UF UF KE KE A:SSS: z 1I ! 1Iq !ayD1 V  !V%MfX#VV !ayA~%1*{{Avay>!((%--PP aEQJ{{5vayk!9:188SXS`S`aa zck( AYq "E188We T ! ( AYq 6!9 -uahh T A:666: 88==F "" " 88==F "" " 77ejj  77ejj  99   a5G   a50 00 0 7g  r fRj l WX^W__bcibjkl eG((K JJ!((#  % % ' 3 3AHH =C 1:D 1:D 3J 1 12I 1 12IZ../F ,I  $ $ 9% 4  4  zz*"3"34HC #J  * 1 1# 6J  * 1 1# 6J   $ $ 3 4  4              JJy ! JJt  JJt  k GY : ==rct|j}||j|f}n|d}|.t|d|j|j||d|\}}n|d|f}|d} t | dk(r1t j| d} t j| d} n6t j| d| dz} t j| d} t||g|dk(rZ|r,tjt|t|| | n\tjt|t|| | n0|dk(rX|r+tjt|t|| | ntjt|t|| | n|dk(rX|r+tjt|t|| | ntj t|t|| | nv|dk(r`|dk(r+tj"t|t|| | nA|dk(rrc|Etj|j|jdf|j|j }|j }|jj|k(sJ|jj|k(sJ|jj|k(sJ|j|jdk(sJ|jrdnd}|j|rdnd}|jd}tjj }t#|j}t#|j} t#|j} t#|} t#|} t%j&|j } t%j&|j}t%j&|j}t%j&|jd}t%j&|}t%j&|}t)|j|j|j||gt+j,||| | | ||||| || t%j.| |S)Nrr&rFT)rCrHr rWrJrUr+r,rmr-rr rrrMr-r?rorGrprr cspmm_coor)rBrrjr+rrrr ptrRowidx ptrColidx ptrValuesrrcnnzcrowsAccolsAccolsBcldbcldcs rspmm_cooroKs {kk YY #AHHAGG  ((C ;;   # %% % ;;   # %% % ;;   # %% % 99 "" "OO-54L ((*MM#y)YfffVZ\`bfhlnpnwnwyEoFG Jrcb|Otj|j|jdf|j|j j }|j}t|j}|jj|k(sJ|jj|k(sJ|j j|k(sJ|j|jdk(s J|jd|j|jrdnd}|j|rdnd}|jd}tj|jd\} } | j!dj#} tj$| d\} } | j#} | j#} | dd ksJd | dd |j tj&tj(fvsJt+| }t+| }t+| }t+|j}t+|j}t+|j }t+|}t+|}t+|}t-j.| j}t-j.|j}t-j.|j}t-j.|j}t-j.|jd}t-j.|jd}t-j.|}t-j.|}t1|j|j|j |||g|j tj&k(r#t3j4||||||||||||||n?|j tj(k(r"t3j6||||||||||||||t9||S) Nrr&rr FTr;) descendingrz)Current max count per row is 8 but found r)rCrr rWrJrrUr+rr,rmr-r rrr?cumsumrrrrrorGrprr cspmm_coo_very_sparse_naive_fp16 cspmm_coo_very_sparse_naive_int8r)rBr dequant_statsrjr+rKrrrrrCr max_countmax_idx ptrOffset ptrMaxCount ptrMaxIdxrfrgrhrrptrDequantStats cnnz_rowsrirjrkcrowsBrlrmrns rspmm_coo_very_sparser~osa {kk YY #AHHDKK*I ::dhh D ZZ "F ZZ "F ZZ #F ZZ #F ::c?D ::c?D t{{DKKamLMww%-- ,,               EJJ  ,,               "k Jrg_@vectorc(|dk(rmtj|jj}tj||z dzj tj }||fS|dvrktjtj||d}tj|t|z zj tj }||fS|dk(r|j}|j}|j|jz }|dk(rd}d |z }|j}tj||z} tj||z| z | z}||fS|d vr|j}|j}tj||dtj||dz }d||dk(<d |z }tj||d}tj||z} tj||z| z | z}||fS|d k(rtj5tj|} tj| |d}|d z}| |j| kD} tj|| } |j| | | z|| <tj||z tzj tj }ddd||fSy#1swYfSxYw) Nlinear)rrT)r>keepdim zeropointrrgo@)vector-zeropoint row-zeropointtruncated-vectorgffffff?)rCrrrGrr ramaxCrUminaminno_grad expand_asr) xr>rprxqrUdynaqxminxzpxabsxr"rs rvectorwise_quantrsXyy|!'') [[TC ( + +EJJ 74x ( (zz%))A,C> [[a$h ( + +EJJ 74x { " GGIuuw  19D T\uuwkk$)$ KKQ % +"u < < GGIzz!d3ejj 37  TQY T\zz!d3kk$)$ KKQ % +"u ) ) ]]_ :99Qrs    /-  $"%":":CQ$,*\0d   .v."++.& '"AF&IX   %P3&3v3e3ek3lF&FFvFSYFDJFV*. H Hvv~&H H  H  H  H HT0hPFPFPPPFPFPPNVNVNNGMN`JfJ5+@JQWJekJ@CJMSJJfJ5+@JQWJekJ@CJMSJKvK5+@KQWKekK@CK_eK\  f & F  *.    vv~&          (&Vv:F&vLZZ ZZ  Z  Z  Z Z Z Z ZZZZZ  !Z\)BB BB  B  B  B B B B B B B B BB !B"#B$%B&'B()B* +Bh#:: ::  :  :  : : : : : : :::: !:$ %:zCD)2 )2#)2+.)2@)X!HSl +\-2JJ88v-2JJ 4r