
    #iD                         S SK r S SKrS SKJr  S SKJrJrJrJrJ	r	  S SK
Jr  S SKJr  S SKJr  S SKJrJrJrJr  S SKJrJrJrJrJr  S S	KJrJrJr   " S
 S5      rS r \!S:X  a  \ " 5         gg)    N)Path)ListDictAnyOptionalTuple)PDFProcessor)VectorStore)RAGAgent)PDF_DIRDOCUMENTS_DIRDEFAULT_RETRIEVAL_KMIN_CHUNK_SCORE)RAGSystemErrorVectorStoreError
ModelErrorDocumentProcessingErrorValidationError)
get_loggerlog_performancelog_error_with_contextc                   f    \ rS rSrS rS rS rSS jrS rS r	S	\
S
\\
   4S jrS	\
S
\4S jrSrg)	RAGSystem   c                     [        [        5      U l        [        5       U l        [        5       U l        S U l        SU l        g )NF)	r   __name__loggerr	   pdf_processorr
   vector_store	rag_agent_initializedselfs    1/var/www/html/leadgen/backtest/airagagent/main.py__init__RAGSystem.__init__   s1     *)^'M!    c                 >    U R                   c  [        5       U l         gg)z6Initialize RAG agent (called on demand to save memory)N)r    r   r"   s    r$   initialize_modelRAGSystem.initialize_model   s     >>!%ZDN "r'   c           	         U R                   R                  S5        [        R                  " 5       n U R                  R	                  5         U R                  5         U R                  R                  5       nU(       a  U R                   R                  S[        U5       S35        U R                  5         U R                  R                  U5        U R                  R                  5         U R                   R                  S[        U5       S35        U R                  R                  (       a  [        U R                  R                  5      OSn[        R                  " 5       U-
  n[        U R                   SUUU(       a  [        U5      OSS9  U R                   R                  S	U S
35        U$ ! [         a   n[        U R                   USS0S5        e SnAff = f)zInitialize the RAG systemz!RAG System initialization startedzProcessing z new document chunkszSuccessfully processed 0 raw chunks (enrichment available via UI button)r   system_setup)total_documentsnew_chunks_processedz)RAG System initialized successfully with z
 documents	operationsetup_systemzFailed to initialize RAG systemN)r   infotimer   load_existing_indexensure_enriched_indexr   process_all_newlenr)   add_documents
save_index	documentsr   	Exceptionr   )r#   
start_time
new_chunks
total_docssetup_durationes         r$   r1   RAGSystem.setup_system    s|   <=YY[
	113&&( ++;;=J   ;s:.??S!TU%%'!!//
;!!,,.  #:3z?:KK{!|}=A=N=N=X=XT..889^_J!YY[:5NDKK*4BLs:RSU KKHT^_` 	"4;;K3PRst	s   E=F0 0
G:GGNc                    [         R                   " 5       nUb  [        U5      SS OSnU R                  R                  SU U(       a  [	        [        U5      5      S:  a  SOS 35        Uc  [
        nU(       d   U R                  R                  S5        S/ S	.$ [        U[        5      (       d,  U R                  R                  S
[        U5       35        S/ S	.$ UR                  5       nU(       d   U R                  R                  S5        S/ S	.$ U R                  R                  (       d   U R                  R                  S5        S/ S	.$  U R                  5         U R                  U5      nU R                  R                  SU 35         U R!                  U5      (       a{  U R                  R"                  R%                  USS9nU Vs/ s H  oS   PM	     n	nU R                  R'                  XUS9n
U R                  R                  S[	        U	5       S35        O5U R                  R)                  XUS9n
U R                  R                  S5        U R                  R+                  S[	        U
5       S35        U R                  R+                  S[	        U
5       S35        U R                  R                  S[	        U
5       S35        [-        U
SS S5       H  u  pUR/                  S S!5      nUR/                  S"S!5      nUR/                  S#S!5      nUR/                  S$0 5      R/                  S%S&5      nU R                  R                  S'U S(USS)  S*US+ S,US+ S-US+ S.35        M     U
 Vs/ s H!  nUR/                  S S!5      [0        :  d  M  UPM#     nnU(       aw  U R                  R                  S/[	        U5       S0[0         S.35        USS1  Vs/ s H  nUR/                  S S!5      S+ PM     nnU R                  R                  S2U 35        Un
OtU
(       am  U R                  R                  S3[0         S435        U
SS5  Vs/ s H  nUR/                  S S!5      S+ PM     nnU R                  R                  S6U 35        U
SS5 n
U
(       d&  U R                  R                  S=USS  35        S>/ S	.$  U R                  R+                  S?[	        U
5       S35        U R8                  R;                  XS@9nU R                  R+                  SA[	        U5       SB35        U R8                  R=                  UX5      nU R8                  R?                  U
5      nU R                  R                  SC[	        U5       SD35        [-        USS1 S5       HN  u  nnU R                  R                  SEU S(UR/                  S%S&5      SS)  SFUR/                  S S!5      S+ 35        MP     [         R                   " 5       U-
  n[A        U R                  SGU[	        U5      [	        U
5      U(       a  [	        U5      OSHSI9  U R                  R                  SJUSK SL35        UUS	.$ ! [         a8  n[        U R                  USUSS S.5        S[        U5       3/ S	.s SnA$ SnAff = fs  snf s  snf s  snf s  snf ! [2         aD  nU R                  R                  S7UR4                   35        S8UR4                   3/ S	.s SnA$ SnAf[6         a9  n[        U R                  US9USS S.5        S:UR4                   3/ S	.s SnA$ SnAf[         a8  n[        U R                  US;USS S.5        S<[        U5       3/ S	.s SnA$ SnAff = f! [         a8  n[        U R                  USMUSS S.5        SN[        U5       3/ S	.s SnA$ SnAff = f)Oz1Search for relevant documents and generate answerNd   NonezProcessing question: z... zEmpty question providedzPlease provide a question.)answersourceszInvalid question type: zQuestion must be a string.zQuestion is only whitespacez%Please provide a meaningful question.z+Search attempted but no documents availablezANo documents available in the system. Please add some PDFs first.model_initialization)r0   questionzError initializing model: zDetected domain: 
   max_keywordskeyword)kdomainz Used keyword-focused search for z	 keywordszUsed enhanced general searchzFound z search resultsz
Retrieved z chunks with scores:   scoreg        vector_scorekeyword_scoremetadatasourceUnknownz  Chunk : 2   z - Combined: .3fz
 (Vector: z, Keyword: )zFiltered to z high-quality chunks (score >=    zFiltered chunk scores: zAll chunks below threshold z, using top 3 anyway   zTop 3 scores: zQuery validation failed: zInvalid search query: vector_searchzVector store error: searchzUnexpected search error: z*No relevant documents found for question: z.No relevant documents found for your question.zFormatting context from )rI   zContext length: z characterszGenerated answer with z sourcesz	  Source z
 - Score: question_answerr   )question_lengthresults_countanswer_lengthz"Successfully answered question in z.2fsanswer_generationzError generating answer: )!r3   strr   r2   r7   r   warning
isinstancetypestripr   r:   r)   r;   r   _detect_domain_from_question"_should_use_keyword_focused_searchkeyword_managerextract_keywordskeyword_focused_searchenhanced_searchdebug	enumerategetr   r   messager   r    format_contextgenerate_answerprepare_source_summariesr   )r#   rI   rN   r<   question_previewr@   rO   keywordskwkeyword_listsearch_resultsiresultrQ   rR   rS   rU   rfiltered_results
score_listtop3_scorescontextrF   rG   durations                            r$   search_and_answerRAGSystem.search_and_answerE   s   YY[
2:2F3x=#.F01A0BHY\]`ai]jYknqYq5wyBz{| 9#A KK 9:6  (C((KK"9$x.9I JK6  >>#KK =>A 
   **KK MN] 	!!# 228<,VH567	66x@@,,<<MMhegMh8@A"9A!%!2!2!I!I,dj!I!k  #CCDUCVV_!`a "&!2!2!B!B8Y_!B!`  !?@KKs>':&;?KLKKs>':&;?KL KKz#n*=)>>RST&~cr':A>	

7C0%zz.#> &

?C @J377)L  8A3b]5QT+U_`lmp_qq|  ~K  LO  }P  PQ  "R  S ? ,:d>aQUU7C=PTc=c>d  <4D0E/FFefuevvw!xyDTUWVWDXYDXqw!4S 9:DX
Y  #::,!GH!1##&A/ARRf$ghESTVUVEWXEW!%%"5c!:;EWX##n[M$BC!/!3( KKI(SWTW.IYZ[J 	KK 8^9L8M_]^nn33N3VGKK 0WkJK^^33GXVFnn==nMGKK5c'l^8LM&wr{A6	6  9QCr&**Xy2QRUSU2V1WWabhblblmtvybz{~a  "A  B 7 yy{Z/HDKK):H*-h-(+N(;7=F1F
 KKA(3qQR " }  	"4;;AWemnrores3tu6s1vh? 	   B*  e Z Y  	KK";AII; GH2199+>    	"4;;^fgkhk^l3mn0<   	"4;;W_`dadWe3fg5c!fX> 	J  	"4;;ATbjkolobp3qr5c!fX> 	s   Y >Z6 Z"FZ6 1Z'Z'?Z6 Z,6"Z6 1Z6 
Z1(%Z6 ;F!^ 
Z'-ZZZ"Z6 6
^ 9[?9^?^.] :^ ^-^ :^ ^
_
-_?_
_
c                    U R                   R                  (       a  [        U R                   R                  5      S:  a  0 n [        U R                   R                  5      S:  a)  U R                   R                  S   R                  S0 5      nUR                  S5      (       d  UR                  S5      (       a  gU R                  R                  S[        U R                   R                  5       S35        g[        [        [        5      R                  S	5      5      nU(       d  g[        S
5         U R                  U5        g! [        [
        4 a*  nU R                  R                  SU 35        0 n SnANSnAff = f! [         a  n[        SUR                    35        e SnAf["         a  n[        SU 35        e SnAff = f)zMEnsure the vector store uses enriched knowledge cards rather than raw chunks.r   rT   z"Could not access sample metadata: Nsummarycard_idzVector store has zB documents (not enriched). Skipping enrichment for faster startup.z*_chunks.jsonz0Rebuilding knowledge index from stored chunks...zVector store rebuild failed: z!Unexpected error during rebuild: )r   r:   r7   rr   
IndexErrorAttributeErrorr   rp   r2   listr   r   globprint!_rebuild_vector_store_from_chunksr   rs   r;   )r#   sample_metadatar@   chunk_filess       r$   r5   RAGSystem.ensure_enriched_index   s    &&3t/@/@/J/J+Ka+O !O%t((223a7&*&7&7&A&A!&D&H&HUW&XO ""9--1D1DY1O1O   #4S9J9J9T9T5U4V  WY  "Z  [ 4.33ODE@A	22;?' / %!!$Fqc"JK"$%(   	1!))=> 	5aS9:	s=   AE 3F E? E::E?
GF%%G2GGc                    / nU HU  n [        US5       n[        R                  " U5      n[        U[        5      (       a  UR                  U5        SSS5        MW     U(       d  [        S5        g[        S[        U5       S35         [        5       U l
        U R                  5         U(       aN  U R                  R                  U5        U R                  R                  5         [        S[        U5       S	35        gg! , (       d  f       GM
  = f! [         a  n[        SU SU 35         SnAGM/  SnAff = f! [         a  n[        S
UR                   35        e SnAf[          a  n[        SUR                   35        e SnAf["         a  n[        SUR                   35        e SnAf[         a  n[        SU 35        e SnAff = f)zLRebuild the vector store using knowledge cards generated from stored chunks.r~   Nz#Warning: Failed to load chunk file rW   z'No chunk data available for rebuilding.u      • Loaded z stored chunksu!      • Rebuilt vector store with r,   u      • Vector store error: u      • Model error: u"      • Document processing error: u1      • Unexpected error rebuilding vector store: )openjsonloadrg   r   extendr;   r   r7   r
   r   r)   r8   r9   r   rs   r   r   )r#   r   
all_chunks
chunk_filefdatar@   s          r$   r   +RAGSystem._rebuild_vector_store_from_chunks   s   
%JO*c*a99Q<D!$--"))$/ +* & ;<s:/~>?	 +D!!#!!//
;!!,,.9#j/9JJz{| % +*  O;J<r!MNNO$   	/		{;< 	(45& 	6qyykBC 	EaSIJ	sq   D=C?D
A3D< ?
D		DD
D9D44D9<
GEG,FGF++G8GGrI   returnc                   ^ UR                  5       m/ SQ/ SQ/ SQ/ SQS.n0 nUR                  5        H&  u  pE[        U4S jU 5       5      nUS:  d  M"  XcU'   M(     U(       a'  [        UR                  5       S S	9nUS
   S
:  a  US   $ g)z8Detect the domain from the question for targeted search.)footballsoccer
basketballbaseballhockeytennisgolfnflnbamlbnhlzpremier leaguezla liga
bundesligaoddsspread	moneylinez
over/underparlayzsports betting)bitcoinbtcethereumethcryptocurrencycrypto
blockchainminingstakingdefinftaltcointokenwalletexchange)stockequitysharesnasdaqnysedividendearningsipooptionsfuturestrading	investingz
market cap)forexcurrencyfxzeur/usdzgbp/usdzusd/jpypipleveragezcarry tradezcentral bankfedecbzinterest rate)sportsr   stocksr   c              3   6   >#    U  H  oT;   d  M
  S v   M     g7f)rP   N ).0patternquestion_lowers     r$   	<genexpr>9RAGSystem._detect_domain_from_question.<locals>.<genexpr>B  s     Og5Ns   		r   c                     U S   $ )NrP   r   )xs    r$   <lambda>8RAGSystem._detect_domain_from_question.<locals>.<lambda>H  s    1Q4r'   )keyrP   N)loweritemssummax)	r#   rI   domain_patternsdomain_scoresrO   patternsrQ   best_domainr   s	           @r$   rj   &RAGSystem._detect_domain_from_question&  s    !)

*  / 5 5 7FOOOEqy(-f% !8 m113HK1~""1~%r'   c                 4  ^ U R                   R                  R                  TSS9nU Vs/ s H  o3S   S:  d  M  UPM     nn[        U5      S:  n[	        U4S jS 5       5      n[        TR                  5       5      S:  nU=(       d    U=(       d    U$ s  snf )	z5Determine if query should use keyword-focused search.   rK   rQ   gffffff?r\   c              3   H   >#    U  H  oTR                  5       ;   v   M     g 7fN)r   )r   termrI   s     r$   r   ?RAGSystem._should_use_keyword_focused_search.<locals>.<genexpr>Z  s%       C
$X^^%55 C
s   ")
zkelly criterionzbollinger bandsrsimacd	fibonaccir   zround robin	arbitragehedger      )r   rl   rm   r7   anysplit)r#   rI   rx   ry   high_relevance_keywordskeyword_heavyspecific_terms
long_querys    `      r$   rk   ,RAGSystem._should_use_keyword_focused_searchN  s     $$44EEh]_E`08"N"wK#<M2"N 349  C
   )*R/
<<*< #Os
   BB)r!   r   r   r    r   r   )r   
__module____qualname____firstlineno__r%   r)   r1   r   r5   r   re   r   rj   boolrk   __static_attributes__r   r'   r$   r   r      sP    "(#JQf$L&P&S &Xc] &P=3 =4 =r'   r   c            	      r   [        5       n U R                  5       nUS:X  a  [        S5        g[        SU 35          [        S5      R	                  5       nUR                  5       S;   a  gU(       d  M9  [        S5        U R                  U5      n[        SUS	    S
35        US   (       aP  [        S5        [        US   S5       H2  u  pE[        U SUS    SUS   S S35        [        SUS    S
35        M4     M  ! [         a    [        S5         g[         a  n[        SU 35         SnAN5SnAff = f)z"Command line interface for testingr   zDNo documents available. Please add PDFs to the pdf_directory folder.Nz"
Total documents in vector store: z*
Enter your question (or 'quit' to exit): )quitexitqz"Searching and generating answer...z	
Answer: rF   
rG   zSources:rP   z. rU   z	 (Score: rQ   rY   rZ   z   Preview: previewz
Exiting...zError: )
r   r1   r   inputri   r   r   rq   KeyboardInterruptr;   )
rag_system	doc_countrI   r}   r|   rU   r@   s          r$   mainr	  b  sG   J '')IA~TU	/	{
;< 	!JKQQSH~~#886711(;FJvh/034i j!!*6)+<a!@IAQCr&"2!39VG_S<QQRSTL	):(;2>? "A! ( ! 	.! 	!GA3-  	!s*   -C> +C> 4BC> >D6	D6D11D6__main__)"r   r3   pathlibr   typingr   r   r   r   r   airagagent.pdf_processorr	   airagagent.vector_storer
   airagagent.mistral_integrationr   airagagent.configr   r   r   r   airagagent.exceptionsr   r   r   r   r   airagagent.logging_configr   r   r   r   r	  r   r   r'   r$   <module>r     s]       3 3 1 / 3 Z Z  Z YN= N=`
&!P zF r'   