9i ddlmZesdevrddlmZnddlZ ddlZdZ dZ dZ d Z Gd d eZGd d eZej$eGddeZej(eGddeZej,eGddeZej0edZdZGddeZej8eGddeZej<edZddl Z ddl!Z!ddlZddl"Z"ddl#Z$ddl%m&Z&ddl%m'Z'dZ(dZ)ejTZ+ejTZ,e-edej\e-edej\ej^e_0ejbe_2d D] Z3e)ee3 e(ee(ee(eeZ4eZ5dd!l6m7Z7ee"jpjse:e$jvjyd"d#Gd$d%eZ=y#e $rddlZYwxYw)&) version_info.)_sentencepieceNc d|jjz}d|jj d|jj d|dS#tj$rd}YKwxYw)Nz proxy of )this__repr__ __builtin__ Exception __class__ __module____name__)selfstrthiss `/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/sentencepiece/__init__.py _swig_reprrs_ 2 2 44"^^668O8OQX ZZ  sAA+*A+cfd}|S)Nc|dk(r |||y|dk(r|jj|yt||r.tt t ||t r |||ytd|z)Nr thisownz(You cannot add instance attributes to %s)r ownhasattr isinstancegetattrtypepropertyAttributeError)rnamevaluesets rset_instance_attrzE_swig_setattr_nondynamic_instance_variable..set_instance_attrsi 6> dE " Y  IIMM% T4 ZT D0I8%T dE " !Kd!RS S)r!r"s` r*_swig_setattr_nondynamic_instance_variabler%sT r#cfd}|S)Nct||r%tt||ts |||yt d|z)Nz%You cannot add class attributes to %s)rrrrr)clsrr r!s rset_class_attrz?_swig_setattr_nondynamic_class_variable..set_class_attr)s: 3 jd1CX&N T5 ! !H3!NO Or#r$)r!r)s` r'_swig_setattr_nondynamic_class_variabler*(sP r#cfd}|S)zlClass decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclasscp|j|j|jjSN)r __bases____dict__copy)r( metaclasss rwrapperz$_swig_add_metaclass..wrapper3s's}}cll6G6G6IJJr#r$)r1r2s` r_swig_add_metaclassr31sK Nr#c4eZdZdZeej Zy)_SwigNonDynamicMetazKMeta class to enforce nondynamic attributes (no new attributes) for a classN)rr __qualname____doc__r*r __setattr__r$r#rr5r58sU9$:J:JKKr#r5ceZdZedddZeZdZejZ dZ dZ dZ d Zd Zd Zd Zee ZeeZee ZeeZee ZeeZeeZd ZdZdZeZy)1ImmutableSentencePieceText_ImmutableSentencePiecec6|jjSr-r rxs rz:ImmutableSentencePieceText_ImmutableSentencePiece.>r#c8|jj|Sr-r<r>vs rr?z:ImmutableSentencePieceText_ImmutableSentencePiece.>AFFJJqMr#The membership flagdoccTtj|tjyr-)r:ImmutableSentencePieceText_ImmutableSentencePiece_swiginit5new_ImmutableSentencePieceText_ImmutableSentencePiecers r__init__z:ImmutableSentencePieceText_ImmutableSentencePiece.__init__As,QQRVXfY]Y]Y_ `r#c,tj|Sr-)r8ImmutableSentencePieceText_ImmutableSentencePiece__piecerKs r_piecez8ImmutableSentencePieceText_ImmutableSentencePiece._pieceEVVW[\\r#c,tj|Sr-)r:ImmutableSentencePieceText_ImmutableSentencePiece__surfacerKs r_surfacez:ImmutableSentencePieceText_ImmutableSentencePiece._surfaceHsXXY]^^r#c,tj|Sr-)r5ImmutableSentencePieceText_ImmutableSentencePiece__idrKs r_idz5ImmutableSentencePieceText_ImmutableSentencePiece._idKsSSTXYYr#c,tj|Sr-)r8ImmutableSentencePieceText_ImmutableSentencePiece__beginrKs r_beginz8ImmutableSentencePieceText_ImmutableSentencePiece._beginNrPr#c,tj|Sr-)r6ImmutableSentencePieceText_ImmutableSentencePiece__endrKs r_endz6ImmutableSentencePieceText_ImmutableSentencePiece._endQsTTUYZZr#c,tj|Sr-)rCImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytesrKs r_surface_as_byteszCImmutableSentencePieceText_ImmutableSentencePiece._surface_as_bytesTsaabfggr#c,tj|Sr-)rAImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytesrKs r_piece_as_byteszAImmutableSentencePieceText_ImmutableSentencePiece._piece_as_bytesWs__`deer#cdj|j|j|j|j|j S)Nz3piece: "{}" id: {} surface: "{}" begin: {} end: {} )formatpieceidsurfacebeginendrKs r__str__z9ImmutableSentencePieceText_ImmutableSentencePiece.__str__bs7"6$**dggt||"&**dhh8 8r#c |j|jk(xrj|j|jk(xrO|j|jk(xr4|j|jk(xr|j|jk(Sr-)rerfrgrhrirothers r__eq__z8ImmutableSentencePieceText_ImmutableSentencePiece.__eq__js ZZ5;; &Y477ehh+>Y4<vr@r#c8|jj|Sr-r<rBs rr?z#ImmutableSentencePieceText.vrDr#rErFcTtj|tjyr-)r#ImmutableSentencePieceText_swiginitnew_ImmutableSentencePieceTextrKs rrLz#ImmutableSentencePieceText.__init__ys::4AnAnApqr#c,tj|Sr-)r'ImmutableSentencePieceText__pieces_sizerKs r _pieces_sizez'ImmutableSentencePieceText._pieces_size}sEEdKKr#c.tj||Sr-)r"ImmutableSentencePieceText__piecesrindexs r_piecesz"ImmutableSentencePieceText._piecess@@uMMr#c,tj|Sr-)r ImmutableSentencePieceText__textrKs r_textz ImmutableSentencePieceText._texts>>tDDr#c,tj|Sr-)r!ImmutableSentencePieceText__scorerKs r_scorez!ImmutableSentencePieceText._scores??EEr#c,tj|Sr-)r,ImmutableSentencePieceText_SerializeAsStringrKs rSerializeAsStringz,ImmutableSentencePieceText.SerializeAsStringJJ4PPr#c,tj|Sr-)r)ImmutableSentencePieceText__text_as_bytesrKs r_text_as_bytesz)ImmutableSentencePieceText._text_as_bytessGGMMr#c(eZdZdZdZdZdZeZy)9ImmutableSentencePieceText.ImmutableSentencePieceIteratorcP||_|jj|_yr-)protorlenrrs rrLzBImmutableSentencePieceText.ImmutableSentencePieceIterator.__init__ ::**,r#c|jSr-rrKs r__len__zAImmutableSentencePieceText.ImmutableSentencePieceIterator.__len__ xxr#ct|tr`t|jDcgc]}|jj |c}|j |j|jS|dkr||jz}|dks||jk\r td|jj |Scc}w)Nrzpiece index is out of range) rslicerangerrrstartstopstep IndexErrorrris r __getitem__zEImmutableSentencePieceText.ImmutableSentencePieceIterator.__getitem__s eU #16txxAA$**$$Q'A%++ejjY^YcYcBcd d 19$(("% 19)89 9zz!!%(( B"B?c zdj|Dcgc]}djt|c}Scc}w)N pieces {{ {}}}joinrdrqrr>s rrjzAImmutableSentencePieceText.ImmutableSentencePieceIterator.__str__/yyDIq+223q6:IJJI!8Nrrr6rLrrrjr r$r#rImmutableSentencePieceIteratorr-)Khr#rc,tj|Sr-)rxrrKs rpiecesz!ImmutableSentencePieceText.piecess ' F Ft LLr#cD|j|jk(Sr-rrls rrnz!ImmutableSentencePieceText.__eq__  # # %)@)@)B BBr#c4t|jSr-rprrKs rrrz#ImmutableSentencePieceText.__hash__ $((* ++r#cdj|j|jdj|jDcgc]}djt |c}Scc}w)Nztext: "{}" score: {} {}rr)rdtextscorerrrqrs rrjz"ImmutableSentencePieceText.__str__sXF499djj99PTP[P[%\1&7&>&>s1v&F%\]__&]s!A'N)rrr6rrrr rLr!delete_ImmutableSentencePieceTextrtrrrrrrr text_as_bytesrrrrnrrrjr$r#rrxrxus-/IOdeGHr%GGLNEFQN E?D^,M V E,MMC,_ Hr#rxceZdZedddZeZdZejZ dZ dZ dZ Gd d Zed Zd Zd ZdZeZy)ImmutableNBestSentencePieceTextc6|jjSr-r<r=s rr?z(ImmutableNBestSentencePieceText.r@r#c8|jj|Sr-r<rBs rr?z(ImmutableNBestSentencePieceText.rDr#rErFcTtj|tjyr-)r(ImmutableNBestSentencePieceText_swiginit#new_ImmutableNBestSentencePieceTextrKs rrLz(ImmutableNBestSentencePieceText.__init__s??nFxFxFz{r#c,tj|Sr-)r,ImmutableNBestSentencePieceText__nbests_sizerKs r _nbests_sizez,ImmutableNBestSentencePieceText._nbests_sizerr#c.tj||Sr-)r'ImmutableNBestSentencePieceText__nbestsrs r_nbestsz'ImmutableNBestSentencePieceText._nbestssEEdERRr#c,tj|Sr-)r1ImmutableNBestSentencePieceText_SerializeAsStringrKs rrz1ImmutableNBestSentencePieceText.SerializeAsStringsOOPTUUr#c(eZdZdZdZdZdZeZy)BImmutableNBestSentencePieceText.ImmutableSentencePieceTextIteratorcP||_|jj|_yr-)rrrrs rrLzKImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__init__rr#c|jSr-rrKs rrzJImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__len__rr#ct|tr`t|jDcgc]}|jj |c}|j |j|jS|dkr||jz}|dks||jk\r td|jj |Scc}w)Nrznbests index is out of range) rrrrrrrrrrrs rrzNImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__getitem__s eU #16txxAA$**$$Q'A%++ejjY^YcYcBcd d 19$(("% 19)9: :zz!!%(( Brc zdj|Dcgc]}djt|c}Scc}wNrznbests {{ {}}}rrs rrjzJImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator.__str__rrNrr$r#r"ImmutableSentencePieceTextIteratorrrr#rc,tj|Sr-)rrrKs rnbestsz&ImmutableNBestSentencePieceText.nbestss , O OPT UUr#cD|j|jk(Sr-rrls rrnz&ImmutableNBestSentencePieceText.__eq__rr#c4t|jSr-rrKs rrrz(ImmutableNBestSentencePieceText.__hash__rr#c dj|jDcgc]}djt|c}Scc}wr)rrrdrqrs rrjz'ImmutableNBestSentencePieceText.__str__s3 YY$++NQ)00Q8N OONs!AN)rrr6rrrr rLr&delete_ImmutableNBestSentencePieceTextrtrrrrrrnrrrjr$r#rrrst-/IOdeGH|%LLQSV,VVC,PHr#rc eZdZedddZeZdZejZ dZ dZ dZ d Zd Zd Zd Zd ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"dZ#dZ$d Z%d!Z&d"Z'd#Z(d$Z)d%Z*d&Z+d'Z,d(Z-d)Z.d*Z/d+Z0d,Z1d-Z2d.Z3d/Z4d0Z5d1Z6d2Z7d3Z8d4Z9d5Z:d6Z;d7Zd:Z?d;Z@d<ZAd=ZBd>d>eCd?d?d?d?d?d@dAd@f dBZD dgdCZEdDZFdEZGdFZHdGZIdhdHZJdhdIZKdhdJZLdhdKZM didLZNdjdMZOdjdNZPdjdOZQdjdPZR dgdQZSdhdRZTdhdSZUdhdTZVdhdUZWeXd>fdVZYeXfdWZZeXfdXZ[dkdYZ\dkdZZ]dld[Z^dld\Z_djd]Zdjd^Z`d_Zad`ZbdaZcdbZddcZeddZfdeZgdhdfZhy>)mSentencePieceProcessorc6|jjSr-r<r=s rr?zSentencePieceProcessor.r@r#c8|jj|Sr-r<rBs rr?zSentencePieceProcessor.rDr#rErFcTtj|tjyr-)rSentencePieceProcessor_swiginitnew_SentencePieceProcessorrKs rrLzSentencePieceProcessor.__init__s66t^=f=f=hir#c.tj||Sr-)r.SentencePieceProcessor_LoadFromSerializedProtor serializeds rLoadFromSerializedProtoz.SentencePieceProcessor.LoadFromSerializedProtosLLTS]^^r#c.tj||Sr-)r,SentencePieceProcessor_SetEncodeExtraOptionsr extra_options rSetEncodeExtraOptionsz,SentencePieceProcessor.SetEncodeExtraOptionsJJ4Q]^^r#c.tj||Sr-)r,SentencePieceProcessor_SetDecodeExtraOptionsrs rSetDecodeExtraOptionsz,SentencePieceProcessor.SetDecodeExtraOptionsrr#c.tj||Sr-)r$SentencePieceProcessor_SetVocabulary)r valid_vocabs r SetVocabularyz$SentencePieceProcessor.SetVocabularysBB4UUr#c,tj|Sr-)r&SentencePieceProcessor_ResetVocabularyrKs rResetVocabularyz&SentencePieceProcessor.ResetVocabulary sDDTJJr#c0tj|||Sr-)r%SentencePieceProcessor_LoadVocabulary)rfilename thresholds rLoadVocabularyz%SentencePieceProcessor.LoadVocabularysCCD(T]^^r#c.tj|g|Sr-)r'SentencePieceProcessor_CalculateEntropyrargss rCalculateEntropyz'SentencePieceProcessor.CalculateEntropysEEdRTRRr#c,tj|Sr-)r#SentencePieceProcessor_GetPieceSizerKs r GetPieceSizez#SentencePieceProcessor.GetPieceSizesAA$GGr#c.tj||Sr-)r SentencePieceProcessor_PieceToIdrres r PieceToIdz SentencePieceProcessor.PieceToIds>>tUKKr#c.tj||Sr-)r SentencePieceProcessor_IdToPiecerrfs r IdToPiecez SentencePieceProcessor.IdToPiece>>tRHHr#c.tj||Sr-)rSentencePieceProcessor_GetScorer s rGetScorezSentencePieceProcessor.GetScore==dBGGr#c.tj||Sr-)r SentencePieceProcessor_IsUnknownr s r IsUnknownz SentencePieceProcessor.IsUnknown rr#c.tj||Sr-)r SentencePieceProcessor_IsControlr s r IsControlz SentencePieceProcessor.IsControl#rr#c.tj||Sr-)rSentencePieceProcessor_IsUnusedr s rIsUnusedzSentencePieceProcessor.IsUnused&rr#c.tj||Sr-)rSentencePieceProcessor_IsByter s rIsBytezSentencePieceProcessor.IsByte)s;;D"EEr#c,tj|Sr-)rSentencePieceProcessor_unk_idrKs runk_idzSentencePieceProcessor.unk_id,;;DAAr#c,tj|Sr-)rSentencePieceProcessor_bos_idrKs rbos_idzSentencePieceProcessor.bos_id/r#r#c,tj|Sr-)rSentencePieceProcessor_eos_idrKs reos_idzSentencePieceProcessor.eos_id2r#r#c,tj|Sr-)rSentencePieceProcessor_pad_idrKs rpad_idzSentencePieceProcessor.pad_id5r#r#c,tj|Sr-)r-SentencePieceProcessor_serialized_model_protorKs rserialized_model_protoz-SentencePieceProcessor.serialized_model_proto8sKKDQQr#c.tj||Sr-)r#SentencePieceProcessor_LoadFromFilerargs r LoadFromFilez#SentencePieceProcessor.LoadFromFile;sAA$LLr#c <tj||||||||| Sr-)r#SentencePieceProcessor__EncodeAsIds rrenable_sampling nbest_sizealphaadd_bosadd_eosreverseemit_unk_pieces r _EncodeAsIdsz#SentencePieceProcessor._EncodeAsIds>s;AA$o_ikpry|CELN\] ]r#c <tj||||||||| Sr-)r&SentencePieceProcessor__EncodeAsPiecesr7s r_EncodeAsPiecesz&SentencePieceProcessor._EncodeAsPiecesAs<DDT4Q`blnsu|FHOQ_` `r#c <tj||||||||| Sr-)r/SentencePieceProcessor__EncodeAsSerializedProtor7s r_EncodeAsSerializedProtoz/SentencePieceProcessor._EncodeAsSerializedProtoDsAMMdTXZikuw|FHOQXZhi ir#c <tj||||||||| Sr-)r.SentencePieceProcessor__EncodeAsImmutableProtor7s r_EncodeAsImmutableProtoz.SentencePieceProcessor._EncodeAsImmutableProtoGsALLTSWYhjtv{~EGNPWYgh hr#c >tj|||||||||| Sr-)r(SentencePieceProcessor__EncodeAsIdsBatch rins num_threadsr8r9r:r;r<r=r>s r_EncodeAsIdsBatchz(SentencePieceProcessor._EncodeAsIdsBatchJsGFFtSR]_npz}BDKMTV]_mn nr#c >tj|||||||||| Sr-)r+SentencePieceProcessor__EncodeAsPiecesBatchrKs r_EncodeAsPiecesBatchz+SentencePieceProcessor._EncodeAsPiecesBatchMsHII$PSU`bqs}@EGNPWY`bpq qr#c >tj|||||||||| Sr-)r4SentencePieceProcessor__EncodeAsSerializedProtoBatchrKs r_EncodeAsSerializedProtoBatchz4SentencePieceProcessor._EncodeAsSerializedProtoBatchPsMRRSWY\^ikz}GINPWY`bikyz zr#c >tj|||||||||| Sr-)r3SentencePieceProcessor__EncodeAsImmutableProtoBatchrKs r_EncodeAsImmutableProtoBatchz3SentencePieceProcessor._EncodeAsImmutableProtoBatchSsMQQRVX[]hjy|FHMOVX_ahjxy yr#c.tj||Sr-)r!SentencePieceProcessor__DecodeIdsridss r _DecodeIdsz!SentencePieceProcessor._DecodeIdsVs??cJJr#c.tj||Sr-)r(SentencePieceProcessor__DecodeIdsAsBytesrZs r_DecodeIdsAsBytesz(SentencePieceProcessor._DecodeIdsAsBytesYsFFtSQQr#c.tj||Sr-)r$SentencePieceProcessor__DecodePiecesrrs r _DecodePiecesz$SentencePieceProcessor._DecodePieces\sBB4PPr#c.tj||Sr-)r2SentencePieceProcessor__DecodeIdsAsSerializedProtorZs r_DecodeIdsAsSerializedProtoz2SentencePieceProcessor._DecodeIdsAsSerializedProto_sPPQUWZ[[r#c.tj||Sr-)r5SentencePieceProcessor__DecodePiecesAsSerializedProtorbs r_DecodePiecesAsSerializedProtoz5SentencePieceProcessor._DecodePiecesAsSerializedProtobsSSTXZ`aar#c.tj||Sr-)r1SentencePieceProcessor__DecodeIdsAsImmutableProtorZs r_DecodeIdsAsImmutableProtoz1SentencePieceProcessor._DecodeIdsAsImmutableProtoesOOPTVYZZr#c.tj||Sr-)r4SentencePieceProcessor__DecodePiecesAsImmutableProtorbs r_DecodePiecesAsImmutableProtoz4SentencePieceProcessor._DecodePiecesAsImmutableProtohsRRSWY_``r#c0tj|||Sr-)r&SentencePieceProcessor__DecodeIdsBatchrrLrMs r_DecodeIdsBatchz&SentencePieceProcessor._DecodeIdsBatchksDDT3P[\\r#c0tj|||Sr-)r-SentencePieceProcessor__DecodeIdsAsBytesBatchrrs r_DecodeIdsAsBytesBatchz-SentencePieceProcessor._DecodeIdsAsBytesBatchnsKKDRUWbccr#c0tj|||Sr-)r7SentencePieceProcessor__DecodeIdsAsSerializedProtoBatchrrs r _DecodeIdsAsSerializedProtoBatchz7SentencePieceProcessor._DecodeIdsAsSerializedProtoBatchqsUUVZ\_almmr#c0tj|||Sr-)r6SentencePieceProcessor__DecodeIdsAsImmutableProtoBatchrrs r_DecodeIdsAsImmutableProtoBatchz6SentencePieceProcessor._DecodeIdsAsImmutableProtoBatchtsTTUY[^`kllr#c0tj|||Sr-)r)SentencePieceProcessor__DecodePiecesBatchrrs r_DecodePiecesBatchz)SentencePieceProcessor._DecodePiecesBatchwsGGcS^__r#c0tj|||Sr-)r:SentencePieceProcessor__DecodePiecesAsSerializedProtoBatchrrs r#_DecodePiecesAsSerializedProtoBatchz:SentencePieceProcessor._DecodePiecesAsSerializedProtoBatchzsXXY]_bdoppr#c0tj|||Sr-)r9SentencePieceProcessor__DecodePiecesAsImmutableProtoBatchrrs r"_DecodePiecesAsImmutableProtoBatchz9SentencePieceProcessor._DecodePiecesAsImmutableProtoBatch}sWWX\^acnoor#c 8tj|||||||Sr-)r(SentencePieceProcessor__NBestEncodeAsIdsrrr9r;r<r=r>s r_NBestEncodeAsIdsz(SentencePieceProcessor._NBestEncodeAsIdss.FFtTS]_fhoqx{IJ Jr#c 8tj|||||||Sr-)r+SentencePieceProcessor__NBestEncodeAsPiecesrs r_NBestEncodeAsPiecesz+SentencePieceProcessor._NBestEncodeAsPiecess/II$PTV`bikrt{~LM Mr#c 8tj|||||||Sr-)r4SentencePieceProcessor__NBestEncodeAsSerializedProtors r_NBestEncodeAsSerializedProtoz4SentencePieceProcessor._NBestEncodeAsSerializedProtos4RRSWY]_ikrt{~EGUV Vr#c 8tj|||||||Sr-)r3SentencePieceProcessor__NBestEncodeAsImmutableProtors r_NBestEncodeAsImmutableProtoz3SentencePieceProcessor._NBestEncodeAsImmutableProtos4QQRVX\^hjqsz}DFTU Ur#c >tj|||||||||| Sr-)r1SentencePieceProcessor__SampleEncodeAndScoreAsIds rr num_samplesr:wor include_bestr;r<r=r>s r_SampleEncodeAndScoreAsIdsz1SentencePieceProcessor._SampleEncodeAndScoreAsIdssIOOPTVZ\ginpsvBDKMTV]_mn nr#c >tj|||||||||| Sr-)r4SentencePieceProcessor__SampleEncodeAndScoreAsPiecesrs r_SampleEncodeAndScoreAsPiecesz4SentencePieceProcessor._SampleEncodeAndScoreAsPiecessIRRSWY]_jlqsvyEGNPWY`bpq qr#c >tj|||||||||| Sr-)r=SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProtors r&_SampleEncodeAndScoreAsSerializedProtoz=SentencePieceProcessor._SampleEncodeAndScoreAsSerializedProtosI[[\`bfhsuz|BNPWY`bikyz zr#c >tj|||||||||| Sr-)r to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. nbest_size = {0,1}: No sampling is performed. nbest_size > 1: samples from the nbest_size results. nbest_size < 0: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. alpha: Soothing parameter for unigram sampling, and dropout probability of merge operations for BPE-dropout. num_threads: number of threads in batch processing (Default = -1, auto-detected) ) model_file model_protoN) $_sentencepiece_processor_init_native _out_type_add_bos_add_eos_reverse_emit_unk_piece_enable_sampling _nbest_size_alpha _num_threadsLoad) rrrout_typer;r<r=r>r8r9r:rMs rInitzSentencePieceProcessor.InitsjD+40dndmdmdm+d-d#ddk%d { Z[ A#r#c | |j}| |j}| |j}| |j}| |j}| |j }| |j }| |j} | |j} |dk(r| |dk(s|dk(s| td| t| tur tdt|tur|tur|j|| ||| |||| S|tur|j|| ||| |||| S|dk(s|dk(r|j!|| ||| |||| S|dk(r|j#|| ||| |||| S|tur|j%|||| ||||S|tur|j'|||| ||||S|dk(s|dk(r|j)|||| ||||S|dk(r|j+|||| ||||Std j-|) a~Encode text input to segmented ids or tokens. Args: input: input string. accepsts list of string. out_type: output type. int or str. add_bos: Add to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. nbest_size = {0,1}: No sampling is performed. nbest_size > 1: samples from the nbest_size results. nbest_size < 0: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. alpha: Soothing parameter for unigram sampling, and merge probability for BPE-dropout (probablity 'p' in BPE-dropout paper). num_threads: the number of threads used in the batch processing (Default = -1). TrraWhen enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. when "nbest_size = -1" , this method samples from all candidates on the lattice instead of nbest segmentations.num_threads must be intserialized_protorimmutable_protozunknown out_type={})rrrrrrrrr RuntimeErrorrintlistrNrqrQrTrWr?rBrErHrd) rinputrr;r<r=r>r8r9r:rMs rEncodezSentencePieceProcessor.Encodes@  >> -- -- --  --  //  %%    '' D j&8J!O&0Ao .    [ 1 <455 e  s?''{OZ(-w.Z Z s?**5+PZ+0'7G^] ] ) )X-@33E;Yc497GWVdf f ( (225+Xb38'7GUce e S  !&'>S S S##E?J$)7GWnV V ' '8w+>,,UOZ-2GWg~_ _ & &++E?J,17GWn^ ^ .55h? @@r#c 4|jd|td|SNrrr$rrqrrkwargss rEncodeAsPiecesz%SentencePieceProcessor.EncodeAsPieces5 T[[ =us =f ==r#c 4|jd|td|Srrrrs r EncodeAsIdsz"SentencePieceProcessor.EncodeAsIds9rr#c ,|jd|dd|S)Nrrr$rrs rEncodeAsSerializedProtoz.SentencePieceProcessor.EncodeAsSerializedProto=s T[[ Lu/A LV LLr#c ,|jd|dd|S)Nrrr$rrs rEncodeAsImmutableProtoz-SentencePieceProcessor.EncodeAsImmutableProtoAs T[[ Ku/@ KF KKr#c :|jd|||tdd|SNTrr9r:rr8r$rrrr9r:rs rSampleEncodeAsPiecesz+SentencePieceProcessor.SampleEncodeAsPiecesE1 T[[Gu5"%tG?EGGr#c :|jd|||tdd|Srrrs rSampleEncodeAsIdsz(SentencePieceProcessor.SampleEncodeAsIdsJrr#c 2|jd|||ddd|S)NrTrr$rrs rSampleEncodeAsSerializedProtoz4SentencePieceProcessor.SampleEncodeAsSerializedProtoOs1 T[[Vu5"4dVNTVVr#c 2|jd|||ddd|S)NrTrr$rrs rSampleEncodeAsImmutableProtoz3SentencePieceProcessor.SampleEncodeAsImmutableProtoTs1 T[[Uu5"3TUMSUUr#cD j j j j j j dkrdfd}t |tur|D cgc] } ||  c} S||Scc} w)aNBestEncode text input to segmented ids or tokens. Args: input: input string. accepsts list of string. out_type: output type. int or str. add_bos: Add to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) nbest_size: nbest size rrcturj|Sturj|Sdk(sdk(rj |Sdk(rj |St d)Nrrrzunknown out_type)rrrqrrrr)rr;r<r>r9rr=rs r_encodez3SentencePieceProcessor.NBestEncode.._encode}s s?''j(/'>S S s?**4+2GWnV V ) )X-@33D*4;Wg~_ _ ( (2243:GWn^ ^-..r#)rrrrrrrr) rrrr;r<r=r>r9rns ` `````` r NBestEncodez"SentencePieceProcessor.NBestEncodeYs(  >> -- -- --  --  %% q // e $)*q ** U^+sBc 6|jd||td|SNrr9rr$)rrqrrr9rs rNBestEncodeAsPiecesz*SentencePieceProcessor.NBestEncodeAsPieces+ T  6Ej'*6.466r#c 6|jd||td|Sr)rrrs rNBestEncodeAsIdsz'SentencePieceProcessor.NBestEncodeAsIdsrr#c .|jd||dd|S)Nrrr$rrs rNBestEncodeAsSerializedProtoz3SentencePieceProcessor.NBestEncodeAsSerializedProtos/ T  EEj'9E=CEEr#c .|jd||dd|S)Nrrr$rrs rNBestEncodeAsImmutableProtoz2SentencePieceProcessor.NBestEncodeAsImmutableProtos/ T  DEj'8D to the result (Default = false) add_eos: Add to the result (Default = false) / is added after reversing (if enabled). reverse: Reverses the tokenized sequence (Default = false) emit_unk_piece: Emits the unk literal string (Default = false) num_samples: How many samples to return (Default = 1) alpha: inverse temperature for sampling wor: whether to sample without replacement (Default = false) include_best: whether to include the best tokenization, requires wor=True (Default = false) rg?Frznum_examples must be positivez8When include_best is True, We must specify "wor = True".c   tur j|  Stur j|  Sdk(sdk(r j |  Sdk(r j |  St d)Nrrrzunknown output type)rrrqrrrr) rr;r<r:r>rrrr=rrs rrz._encodes s?00{E3P\18'7N\ \ s?33D+ucS_4;Wg~_ _ ) )X-@<rr:rrrrs ` ````````` rSampleEncodeAndScorez+SentencePieceProcessor.SampleEncodeAndScores4  >> -- -- --  --        :;; cUVV22& e $)*q ** U^+s#B=c 8|jd|||td|SNrrr:rr$)rrqrrrr:rs rSampleEncodeAndScoreAsPiecesz3SentencePieceProcessor.SampleEncodeAndScoreAsPieces. &T & &?U SX03?7=??r#c 8|jd|||td|Sr)rrrs rSampleEncodeAndScoreAsIdsz0SentencePieceProcessor.SampleEncodeAndScoreAsIdsrr#c 0|jd|||dd|S)Nrrr$rrs r%SampleEncodeAndScoreAsSerializedProtoz67 7**5%EE  # #E5 11r#czfd}t|tur|Dcgc] }|| c}S||Scc}w)NcLrj|Sj|Sr-rrrr with_offsetss r _normalizez4SentencePieceProcessor.Normalize.._normalize& ++D1 1t$$r#rrrrr"r#r>s` ` r Normalizez SentencePieceProcessor.Normalize;% e ',-! 1 --  .8c ti}|jD]\}}t|||<|j|Sr-)itemsrqr)rr new_kwargskeyr s rOverrideNormalizerSpecz-SentencePieceProcessor.OverrideNormalizerSpecs>j %*#ue* 3%  ) )* 55r#c"|jSr-rrKs r piece_sizez!SentencePieceProcessor.piece_size     r#c"|jSr-r0rKs r vocab_sizez!SentencePieceProcessor.vocab_sizer2r#c"|jSr-r/rKs r __getstate__z#SentencePieceProcessor.__getstate__  ( ( **r#cF|j|j|yr-rLrrr/s r __setstate__z#SentencePieceProcessor.__setstate__ mmo ""#9:r#c"|jSr-r0rKs rrzSentencePieceProcessor.__len__r2r#c$|j|Sr-)r r s rrz"SentencePieceProcessor.__getitem__s ^^E ""r#ch|r |r td|r|j|S|j|S)aOverwride SentencePieceProcessor.Load to support both model_file and model_proto. Args: model_file: The sentencepiece model file path. model_proto: The sentencepiece model serialized proto. Either `model_file` or `model_proto` must be set. z-model_file and model_proto must be exclusive.)rrr4)rrrs rrzSentencePieceProcessor.Loads:  JKK ++K88   z **r#) NNNNNNNNNNN)NNNNNNr-)r)r)irrr6rrrr rLrdelete_SentencePieceProcessorrtrrrrrrrrr rrrrrrr"r&r)r,r/r4r?rBrErHrNrQrTrWr\r_rcrfrirlrorsrvryr|rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rqr rrrrrrr'r.r1r4r7r<rrrr$r#rrrs-/IOdeGHj%CC___VK_SHLIHIIHFBBBBRM]`ihnqzyKRQ\b[a]dnm`qpJMVUnqzyLWZkY!"-Bd"#[|>>MLG G V U"   #'#7t6 6 E D'+%)%)%),0)-#'!%*.JZ? ? N M &)d\~,/C),CCCCC 26!!+; !# +r#rc,tj|Sr-)rSetRandomGeneratorSeed)seeds rrDrDs  0 0 66r#c,tj|Sr-)rSetMinLogLevel)rCs rrGrGs  ( ( ++r#ceZdZedddZdZeZedZ edZ edZ ed Z ed Z edd Zedd Zy )SentencePieceTrainerc6|jjSr-r<r=s rr?zSentencePieceTrainer.r@r#c8|jj|Sr-r<rBs rr?zSentencePieceTrainer.rDr#rErFctd)NzNo constructor defined)r)rrrs rrLzSentencePieceTrainer.__init__s566r#c,tj|Sr-)r%SentencePieceTrainer__TrainFromString)r3s r_TrainFromStringz%SentencePieceTrainer._TrainFromStringsCCCHHr#c,tj|Sr-)r"SentencePieceTrainer__TrainFromMaprs r _TrainFromMapz"SentencePieceTrainer._TrainFromMaps@@FFr#c.tj||Sr-)r#SentencePieceTrainer__TrainFromMap2riters r_TrainFromMap2z#SentencePieceTrainer._TrainFromMap2AA$MMr#c,tj|Sr-)r#SentencePieceTrainer__TrainFromMap3rRs r_TrainFromMap3z#SentencePieceTrainer._TrainFromMap3sAA$GGr#c.tj||Sr-)r#SentencePieceTrainer__TrainFromMap4rVs r_TrainFromMap4z#SentencePieceTrainer._TrainFromMap4rYr#Nc |&t|turtj|Sd}d}d}i}|j D]\}}|dvr|} |dvr|}||||< |r@|rtj ||}ntj |}|j|y|rtj||Stj|S)zDTrain Sentencepiece model. Accept both kwargs and legacy string arg.Nc8t|turztjddk(r t }n t }t j|d}|j|Dcgc] }t|c}|jSt|Scc}w)zEncode value to CSV..rr)lineterminator) rrsysrStringIOBytesIOcsvwriterwriterowrqgetvalue)r frhrCs rrz,SentencePieceTrainer._Train.._encodesu ;$    a A % A A::a3& //51a3q61 2 U 2s#B)sentence_iteratorsentence_reader) model_writer) rrqrIrOr+r_r\writerXrS) r3rrrlrnr,r-r rs r_TrainzSentencePieceTrainer._Trains T#Y#-#44S99 lj +*#u : :#  $ $,#EN*S/ +  ,;;J;LN+-;;JG+;' %44ZARS S%33J? ?r#c tt|5tjdd|i|dddy#1swYyxYw)N)ostreamr3r$) _LogStreamrIrp)r3 logstreamrs rTrainzSentencePieceTrainer.Trains5 i (7##66v6777s.7r-rA)rrr6rrrLrr staticmethodrOrSrXr\r_rprur$r#rrIrIs-/IOdeG7HIIGGNNHHNN**X77r#rIceZdZedddZeZdZejZ dZ dZ dZ d Zd Zd Zd Zd Z ddZddZdZdZy)SentencePieceNormalizerc6|jjSr-r<r=s rr?z SentencePieceNormalizer.r@r#c8|jj|Sr-r<rBs rr?z SentencePieceNormalizer.rDr#rErFcTtj|tjyr-)r SentencePieceNormalizer_swiginitnew_SentencePieceNormalizerrKs rrLz SentencePieceNormalizer.__init__ s77n>h>h>jkr#c.tj||Sr-)r/SentencePieceNormalizer_LoadFromSerializedProtors rrz/SentencePieceNormalizer.LoadFromSerializedProto$sMMdT^__r#c.tj||Sr-)r'SentencePieceNormalizer_LoadFromRuleTSV)rrs rLoadFromRuleTSVz'SentencePieceNormalizer.LoadFromRuleTSV'sEEdHUUr#c.tj||Sr-)r(SentencePieceNormalizer_LoadFromRuleName)rrs rLoadFromRuleNamez(SentencePieceNormalizer.LoadFromRuleName*sFFtTRRr#c,tj|Sr-)r.SentencePieceNormalizer_serialized_model_protorKs rr/z.SentencePieceNormalizer.serialized_model_proto-sLLTRRr#c.tj||Sr-)r$SentencePieceNormalizer_LoadFromFiler2s rr4z$SentencePieceNormalizer.LoadFromFile0sBB4MMr#c.tj||Sr-)r"SentencePieceNormalizer__Normalizers rrz"SentencePieceNormalizer._Normalize3s@@tLLr#c.tj||Sr-)r-SentencePieceNormalizer__NormalizeWithOffsetsrs rrz-SentencePieceNormalizer._NormalizeWithOffsets6sKKDRVWWr#c0tj|||Sr-)r&SentencePieceNormalizer__SetProtoField)rrr s r_SetProtoFieldz&SentencePieceNormalizer._SetProtoField9sDDT4QVWWr#NcBt||r|j|}nG|r|j|}n3|r|j|}n|r|j |}n t d|r7|j d||j d||j d|yy)aInitialzie sentencePieceNormalizer. Args: model_file: The sentencepiece model file path. model_proto: The sentencepiece model serialized proto. rule_tsv: The normalization rule file in TSV format. rule_name: Pre-defined normalization name. add_dummy_prefix: add dummy prefix. escape_whitespaces: escape whitespaces. remove_extra_whitespaces: remove extra whitespaces. zno model is specifiedadd_dummy_prefixescape_whitespacesremove_extra_whitespacesN)%_sentencepiece_normalizer_init_nativer4rrrrr) rrrrule_tsv rule_namerrrstatuss rrzSentencePieceNormalizer.Init<s(,D1 "":. --k: %%h/ &&y1233  .0@A 02DE 68PQ r#czfd}t|tur|Dcgc] }|| c}S||Scc}w)NcLrj|Sj|Sr-r r!s rr#z5SentencePieceNormalizer.Normalize.._normalizecr$r#r%r&s` ` rr'z!SentencePieceNormalizer.Normalizebr(r)c"|jSr-r6rKs rr7z$SentencePieceNormalizer.__getstate__mr8r#cF|j|j|yr-r:r;s rr<z$SentencePieceNormalizer.__setstate__qr=r#)NNNNFFFr-)rrr6rrrr rLrdelete_SentencePieceNormalizerrtrrrr/r4rrrrr'r7r<r$r#rrxrxs-/IOdeGHl%DD`VSSNMXX# %&+$RL+;r#rxc,tj|Sr-)r SetDataDir)data_dirs rrrys  $ $X ..r#)re)rfc8i}|jjD]V\}}tjd|stjdd|j j dd}|||<X|jD]\}}t|||y)z1Added snake_cased method from CammelCased method.z^[A-Z]+z(?._funcs9 Aw#~1q5A$7 2 33 1:r#cpt|tur|Dcgc] }|| c}S||Scc}wr-r%)rr3rrs r _batched_funcz!_batchnize.._batched_funcs8 CyD&) *eD!n ** 4 +s3)rr)rrrrrs @@r _batchnizers) D$ '$   )T=)r#rL)r rrrrrr) __version__ sentencepiece package_datac eZdZddZdZdZy)rsNcr||_|j$tjj|_yyr-)rrrdstderrfilenoorig_stream_fileno)rrrs rrLz_LogStream.__init__s-DL || # 1 1 3d r#c|j]tj|j|_tj |jj |jyyr-)rrosduprorig_stream_dupdup2rrKs r __enter__z_LogStream.__enter__sI ||VVD$;$; r#rs)>rdr_swig_python_version_info __package__rrrbuiltinsr ImportErrorrr%r*r3rr5objectr:>ImmutableSentencePieceText_ImmutableSentencePiece_swigregisterrx'ImmutableSentencePieceText_swigregisterr,ImmutableNBestSentencePieceText_swigregisterr#SentencePieceProcessor_swigregisterrDrGrI!SentencePieceTrainer_swigregisterrx$SentencePieceNormalizer_swigregisterrrrgrimportlib.resources importlibiorerfrrrLrrrrrTokenizer  Detokenizemset_random_generator_seedset_min_log_level_versionrpathrrq resourcesfilesrsr$r#rrs8:#/ "[ L$L 33nNMMNADDP7667QR4f4p<;;<[\J +VJ +\3223IJ7,K76K7^1001EFW;fW;v4334KL/  *"(>'F'F$(?(H(H% ,B,G,GH-D-I-IJ"8"?"?$:$A$A! (A #Q' ( &'$%'(2"! 277<<I//55oFG XYY%sG22 G?>G?