9iLddlZddlmZmZddlmZmZddlmZddl Z ddl Z ddl Z ddlZddl Z ddlZddlZddlmZmZmZddlmZddlZddlmZddlmZmZmZmZddlZdd lmZej>e Z!d ed efd Z"d ed efd Z#d ed efdZ$d ed efdZ%d ed efdZ&defdZ' d6dee(de)de*fdZ+dddde#fdede*de*de*de*deeegeff d Z,d!Z-ddd"e#fd#ed$ede*de*de*deeegeff d%Z.d&Z/d'efd(Z0d)Z1d*Z2d7d+Z3 d8d,e(d-ee(d.ee(edfd/ee(d0ee(d1eee(dfd2eee(d3ee)e(dfd4e(fd5Z4y)9N)Tensordevice)ListCallable)tqdm)DictOptionalUnion)Path)HUGGINGFACE_HUB_CACHE)HfApi hf_hub_urlcached_downloadHfFolder)versionabct||S) Computes the cosine similarity cos_sim(a[i], b[j]) for all i and j. :return: Matrix with res[i][j] = cos_sim(a[i], b[j]) )cos_simrrs d/var/www/html/backtest/airagagent/rag_env/lib/python3.12/site-packages/sentence_transformers/util.pypytorch_cos_simrs 1a=c^t|tjstj|}t|tjstj|}t |j dk(r|j d}t |j dk(r|j d}tjjj|dd}tjjj|dd}tj||jddS)rrpdim) isinstancetorchrtensorlenshape unsqueezenn functional normalizemm transpose)rra_normb_norms rrrs a & LLO a & LLO 177|q KKN 177|q KKN XX * *1q * 9F XX * *1q * 9F 88FF,,Q2 33rct|tjstj|}t|tjstj|}t |j dk(r|j d}t |j dk(r|j d}tj||jddS)z Computes the dot-product dot_prod(a[i], b[j]) for all i and j. :return: Matrix with res[i][j] = dot_prod(a[i], b[j]) rr) r!r"rr#r$r%r&r*r+rs r dot_scorer/4s a & LLO a & LLO 177|q KKN 177|q KKN 88Aq{{1a( ))rct|tjstj|}t|tjstj|}||zj dS)zt Computes the pairwise dot-product dot_prod(a[i], b[i]) :return: Vector with res[i] = dot_prod(a[i], b[i]) )r )r!r"rr#sumrs rpairwise_dot_scorer3HsO a & LLO a & LLO E;;2; rct|tjstj|}t|tjstj|}t t |t |S)zm Computes the pairwise cossim cos_sim(a[i], b[i]) :return: Vector with res[i] = cos_sim(a[i], b[i]) )r!r"rr#r3normalize_embeddingsrs rpairwise_cos_simr6VsS a & LLO a & LLO 2157KA7N OOr embeddingscZtjjj|ddS)z[ Normalizes the embeddings matrix, so that each sentence embedding has unit length rrr)r"r'r(r))r7s rr5r5ds& 88   ( (qa ( @@r sentencesshow_progress_bar batch_sizecJ|j|||d}t|g|i|S)ab Given a list of sentences / texts, this function performs paraphrase mining. It compares all sentences against all other sentences and returns a list with the pairs that have the highest cosine similarity score. :param model: SentenceTransformer model for embedding computation :param sentences: A list of strings (texts or sentences) :param show_progress_bar: Plotting of a progress bar :param batch_size: Number of texts that are encoded simultaneously by the model :param query_chunk_size: Search for most similar pairs for #query_chunk_size at the same time. Decrease, to lower memory footprint (increases run-time). :param corpus_chunk_size: Compare a sentence simultaneously against #corpus_chunk_size other sentences. Decrease, to lower memory footprint (increases run-time). :param max_pairs: Maximal number of text pairs returned. :param top_k: For each sentence, we retrieve up to top_k other sentences :param score_function: Function for computing scores. By default, cosine similarity. :return: Returns a list of triplets with the format [score, id1, id2] T)r:r;convert_to_tensor)encodeparaphrase_mining_embeddings)modelr9r:r;argskwargsr7s rparaphrase_miningrCks2.i;LYcw{|J ' DT DV DDriii dquery_chunk_sizecorpus_chunk_size max_pairstop_kscore_functionc |dz }tj}d}d}tdt||D])} tdt||D] } ||| | |z|| | |z} t j | t |t| dddd\} } | jj} | jj} tt| D]n}t| |D][\}}| |z}| |z}||k7s| |||kDs"|j| ||||f|dz }||k\sG|j}|d}]p,t}g}|jsg|j\}}}t||g\}}||k7r-||f|vr'|j||f|j!|||g|jsgt|dd}|S) a Given a list of sentences / texts, this function performs paraphrase mining. It compares all sentences against all other sentences and returns a list with the pairs that have the highest cosine similarity score. :param embeddings: A tensor with the embeddings :param query_chunk_size: Search for most similar pairs for #query_chunk_size at the same time. Decrease, to lower memory footprint (increases run-time). :param corpus_chunk_size: Compare a sentence simultaneously against #corpus_chunk_size other sentences. Decrease, to lower memory footprint (increases run-time). :param max_pairs: Maximal number of text pairs returned. :param top_k: For each sentence, we retrieve up to top_k other sentences :param score_function: Function for computing scores. By default, cosine similarity. :return: Returns a list of triplets with the format [score, id1, id2] rr1rTFr largestsortedc |dS)Nrxs rz.paraphrase_mining_embeddings..s !A$rkeyreverse)queue PriorityQueueranger$r"topkmincputolist enumerateputgetsetemptyrMaddappend)r7rErFrGrHrIpairs min_score num_addedcorpus_start_idxquery_start_idxscoresscores_top_k_valuesscores_top_k_idx query_itr top_k_idx corpus_itrijentry added_pairs pairs_listscoresorted_isorted_js rr?r?sE& QJE    !EII!!S_6GH1$QJ9IJ 1O#JO_?_$`blm}OPaacbcF49JJvs5RUV\]^V_R`Gaghrv@E5F 1 !1"5"9"9";"B"B"D /335<<> "3v;/ 1 -67G 7R-S 1)Iz')3A(:5AAv"5i"@"Ki"W #6y#A)#LaQR"ST!Q $ 1$)IIKE(-aI 1 1 11,%KJkkmiik q!#QF^( x Xx$8 $K OOXx0 1   uam , kkm EJ rct|i|S)z8This function is deprecated. Use semantic_search instead)semantic_search)rArBs rinformation_retrievalrys D +F ++r query_embeddingscorpus_embeddingsc .t|tjtjfrt j |}n%t|t rt j|}t|jdk(r|jd}t|tjtjfrt j |}n%t|t rt j|}|j|jk7r|j|j}tt|Dcgc]}g}}tdt||D]}tdt||D]} |||||z|| | |z} t j| t|t| dddd\} } | j!j#} | j!j#} tt| D]=} t%| | | | D]&\}}| |z}|| z}||j'||d(?tt|D]"}t)||dd||<||d|||<$|Scc}w) a This function performs a cosine similarity search between a list of query embeddings and a list of corpus embeddings. It can be used for Information Retrieval / Semantic Search for corpora up to about 1 Million entries. :param query_embeddings: A 2 dimensional tensor with the query embeddings. :param corpus_embeddings: A 2 dimensional tensor with the corpus embeddings. :param query_chunk_size: Process 100 queries simultaneously. Increasing that value increases the speed, but requires more memory. :param corpus_chunk_size: Scans the corpus 100k entries at a time. Increasing that value increases the speed, but requires more memory. :param top_k: Retrieve top k matching entries. :param score_function: Function for computing scores. By default, cosine similarity. :return: Returns a list with one entry for each query. Each entry is a list of dictionaries with the keys 'corpus_id' and 'score', sorted by decreasing cosine similarity scores. rrTFrK) corpus_idrtc |dS)NrtrOrPs rrRz!semantic_search..s RST[R\rrS)r!npndarraygenericr" from_numpyliststackr$r%r&rtorXrYrZr[r\ziprcrM)r{r|rErFrHrI_queries_result_listrhrg cos_scorescos_scores_top_k_valuescos_scores_top_k_idxrl sub_corpus_idrtr~query_ididxs rrxrxs&"RZZ$<= ++,<= $d + ;;'78  ! !"a'+55a8#bjj"**%=>!,,->? %t ,!KK(9:#3#:#::+../@/G/GH',S1A-B'CD!2DD C(8$9;KLc %a->)?AR S c '(8YiIi(jl}OP`arPrmstJ=BJJzSVW\^ablmnbo^pSqwxCGPU=V 9 #%9&=&A&A&C&J&J&L ##7#;#;#=#D#D#F "3z?3 c ,/0DY0OQhirQs,tc(M5 0= @I.:H'188y[`9abc c cc$S,-.E#)*=c*BH\fj#kC #6s#;Ae#DC E 1Es> Jctjj|dk7r4tjtjj|dt j |d}|j dk7rFtdj||j tj|jy|d z}t|d 5}|jj d }| t|nd}td |d }|j!dD]0}|s|j#t%||j'|2 dddtj(||j+y#1swY0xYw)z1 Downloads a URL to a given path on disc Texist_ok)streamz1Exception when trying to download {}. Response {})fileN_partwbzContent-LengthB)unittotal unit_scale) chunk_size)ospathdirnamemakedirsrequestsr_ status_codeprintformatsysstderrraise_for_statusopenheadersintr iter_contentupdater$writerenameclose) urlrreqdownload_filepath file_binarycontent_lengthrprogresschunks rhttp_getr s9 wwt" BGGOOD)D9 ,,s4 (C # AHHcoo^eheoeop W   &)+)9:'5'AN#tS$?%%%6 )EE +!!%( ) )II& NN))sAF%-FF  target_devicecj|D]-}t||ts||j|||</|S)z4 send a pytorch batch to a device (CPU/GPU) )r!rr)batchrrTs rbatch_to_devicer&s>6 eCj& )s}5E#J6 Lrc|jj}||tjjk(r|jjS|dz|jjzS)z Gives a full name (package_name.class_name) for a class / object in Python. Will be used to load the correct classes from JSON files .) __class__ __module__str__name__)omodules rfullnamer1sR ;; ! !& ^v!9!99 ;;   C.s ArrSct|SrrrPs rrRz%community_detection..s #a&r)r"r#rrZr$maxrXrrYrr\rcrMr`r]r)r7 thresholdmin_community_sizer;extracted_communities sort_max_size start_idxr top_k_valuesrro new_cluster top_val_large top_idx_largervalunique_communities extracted_ids cluster_id communitynon_overlapped_communitys rcommunity_detectionrTsO Yz/@/@AI/ZAA 22B7ZIM1c*oz: Z )j2HI:V %//,>/M as<() :AAr"i/ 0:!}/A/AM[_/A/`, }$B')3$'M(93z?$KM3=a=3E3E _c3E3d0M=$B')3!$M$8$8$:M J,HCY&&s+ , &,,[9% :( 9>##8>NX\]EM!*+@!A ; I9% #%  5C-'(//4 5 ' (,> >  % %&> ?  !9 : ; 28HRVW rrepo_idrevision cache_dir library_namelibrary_version user_agent ignore_filesuse_auth_tokenreturnc |t}t|tr t|}t }d} t|tr|} n|rt j } |j||| } tjj||jdd} | j} t| D]*\} }|jdk(s| | =| j|n| D]o}|0d}|D]&}t!j |j|s$d}n|r6t#||j| j$}tjj|jj'd}tjj)tjj| |}tj*|d || |||||d }t-j.t0j2t-j.d k\rd|d <t5di|}tjj7|d zsXtj8|d zr| S)z Method derived from huggingface_hub. Adds a new parameters 'ignore_files', which allows to ignore certain files / file-patterns N)rrtoken/rz modules.jsonFT)filenamerr)rrforce_filenamerrrrz0.8.1legacy_cache_layoutz.lockrO)r r!r rr r get_token model_inforrjoinreplacesiblingsr] rfilenamercfnmatchrshasplitrrrparsehuggingface_hub __version__rexistsremove)rrrrrrrr_apirrstorage_folder all_filesrrepofile model_file skip_downloadpatternrrelative_filepathnested_dirnamecached_download_argsrs rsnapshot_downloadrs) )T" N 7D E.#& ""$85QJWW\\7??3,N##I"9- X    /#   X &    &&  #!M' ??:#7#7A$(M   j22Z^^ GGLL**>*>*D*DS*IJ GGLL): ;  NT2'*'/(.$,  . ==44 5w9O O;? !6 76!56 77>>$. ) IIdWn %M&&P r)F )g?rzr)NNNNNNN)5rr"rrtypingrrtqdm.autonotebookrrrrnumpyrrVloggingrr r pathlibr rhuggingface_hub.constantsr r rrrr packagingr getLoggerrloggerrrr/r3r6r5rboolrrCr?ryrxrrrrrrrOrrrs~ !"   ((;HH   8 $v&4v4&4,**F*( & V  P P6 PAVA16')E!%cE)-E"%E:/3/5'-#&KR <V<(+<),<"%<! < '//?/G&H <~,-0-3!#IP =f='-=&)=(+= = %-ff-=v-E$F =@6& /.BZ#(,"&%))-(,-1Q QsmQS$_%Q3- Q c] Q dCo& Q49%Q$T/*Q Qr