
    #i,                        S SK r S SKrS SKrS SKrS SKJr  S SKJr  S SKJr  S SK	J
r
  S SKJrJr  S SKJrJrJrJrJrJrJrJrJr  S SKJrJrJrJrJrJrJr  S S	K J!r!  S S
K"J#r#  S SK$J%r%J&r&   " S S5      r'g)    N)Path)datetime)OrderedDict)combinations)ThreadPoolExecutoras_completed)	LANGUAGE_MODELMAX_NEW_TOKENSTEMPERATURE	DO_SAMPLE	NOTES_DIRKNOWLEDGE_CARD_LLM_BUDGETDIGESTS_DIRCAPSULES_DIRDOCUMENTS_DIR)ListDictAnyTupleOptionalSetIterable)SequenceMatcher)fuzz)call_grok_apigenerate_answer_with_grokc                   F	   \ rS rSrSrSr1 SkrS rS rS r	SS\
S	\S
\S\
4S jjrS\
S\
4S jrS\
S\
4S jrS\
S\4S jrS\
S\
4S jrS\
S\
4S jrS\
S\
S\4S jrS\
S\\
   4S jrSS\
S\S\S\
4S jjrSS\
S\S\
4S jjrSS\S\\\
\4      4S jjr SS \\\\
\4         S!\S\\\
\4      4S" jjrS\
S#\\\
\4      S$\
S\
4S% jrS\
S#\\\
\4      S$\
S\
4S& jr SS\
S\
S \\\\
\4         S\
4S( jjrS) r S* r!S+ r"S\
S\
S\
4S, jr#  SS\
S\
S$\
S-\\
   S.\\\\
\4         S\
4S/ jjr$S0\
S1\
S2\
S$\
S3\
S\
4S4 jr%S0\
S1\
S5\
S$\
S3\
S.\\\
\4      S\
4S6 jr&S\
S1\
S5\
S$\
S\
4
S7 jr'S\
S\
S$\
S\
4S8 jr(S\
S9\
S$\
S\
4S: jr)S;\\
   S\*4S< jr+S\\
\\
\\
\4   4   4   4S= jr,SS>\\\
\4      S?\S\\\
\4      4S@ jjr-SA\\
\4   SB\\
\4   S\
4SC jr. SSD\
SB\\
\4   SE\\/R`                     S\\
\4   4SF jjr1SS\
SG\S\\
   4SH jjr2 SSD\
SI\\
\4   SE\\/R`                     S\\\
\4      4SJ jjr3S\
S\\\
\4      4SK jr4SL\\
\\
\\
\4   4   4   SS'4SM jr5SL\\
\\
\\
\4   4   4   SS'4SN jr6SO\
SP\\\
\4      S\\\
\4      4SQ jr7SL\\
\\
\\
\4   4   4   SS'4SR jr8S\\
\\
\4   4   4SS jr9S\\
\\
\4   4   4ST jr:SU\\
   S\\
   4SV jr;S\\
\\
\4   4   4SW jr<SX\\
\\
\4   4   SY\
SZ\
S[\\
   SO\
SS'4S\ jr=S\
S\\
   4S] jr>SS\
S^\S\
4S_ jjr?S\
S\
4S` jrSa\
S\4Sb jr SS\
S\
S \\\\
\4         S\
4Sc jjr@Sd rASS \\\\
\4         4Se jjrBS\
S\*4Sf jrCSS\\
   4Sg jjrDSO\
S\\\
\4      4Sh jrESSO\
SB\\
\4   Si\S\\
   4Sj jjrFSS\
4Sk jjrGSl\\
\4   S\
4Sm jrHSS\
Sn\\I\
      S!\S\\
   4So jjrJSSp\\
\4   Sn\\I\
      S\
4Sq jjrKS#\\\
\4      S\
4Sr jrLS\
Ss\
S#\\\
\4      S1\
S3\
S$\
St\
S\
4Su jrMSv\
S\N\
\
4   4Sw jrO SSx\I\
   Sy\\
\4   S!\S\\
   4Sz jjrPSS{\
S|\
S}\S\*4S~ jjrQS\
S\
4S jrRS\
S\
4S jrSS;\
S\
4S jrTS\\
   S\*4S jrUS\V\
   S\I\
   4S jrWSO\
S\\
   S\
4S jrXSrYg')RAGAgent   z=An intelligent agent that guides RAG responses with reasoningzBased on the available documents, I can provide information about esoteric and mythological topics including Hindu mythology, hermetic philosophy, and mystical traditions. Please ask a more specific question about the content in the uploaded documents.>S   aanasatbebydoifinisitnoofonorsotoallandanyarebutcandidforhadhashowitsmaynotperthewaswhowhyyesalsobeendoesdoneeachfromhaveintomoremostoversomesuchthanthatthiswerewhatwhenwhomwillwithaboutamongbeingcouldmightothershalltheirtherethesethoseunderwherewhichwhosewouldshouldwithinbetweenwithoutdocument	according	available	documentsc                    S U l         S U l        S U l        [        S[        5      U l        SU l        SU l        / U l        SU l	        0 U l
        0 U l        0 U l        U R                  5         U R                  5       U l        0 U l        g )Nr   T   )model	tokenizerpipelinemaxr   knowledge_card_llm_budgetknowledge_card_llm_usedknowledge_card_llm_enabledshort_term_memorymemory_limitdocument_digeststheme_capsulesentity_graphload_distilled_resourcesbuild_entity_graph_index_chunk_cacheselfs    @/var/www/html/leadgen/backtest/airagagent/mistral_integration.py__init__RAGAgent.__init__/   s    
),Q0I)J&'($*.',.;=9;79%%' 99;AC    c                     U R                  5       U l         U R	                  5       U l        g! [         a  n[        SU 35        0 U l         SnAN;SnAff = f! [         a  n[        SU 35        0 U l         SnAgSnAff = f)z>Load previously distilled document digests and theme capsules.z*Warning: failed to load document digests: Nz(Warning: failed to load theme capsules: )load_document_digestsr~   	Exceptionprintload_theme_capsulesr   )r   es     r   r   !RAGAgent.load_distilled_resourcesB   s    	'$($>$>$@D!
	%"&":":"<D  	'>qcBC$&D!!	'  	%<QC@A"$D	%s+   . A 
AAA
B$A>>Bc                     [        S5        g)z4Load language model - DEPRECATED: Now using Grok APIz5Using Grok Fast API - no local model loading requiredN)r   r   s    r   
load_modelRAGAgent.load_modelP   s     	EFr   prompt
max_tokenstemperaturereturnc                     [        UUUS9nU(       a1  X;   a  U[        U5      S R                  5       $ UR                  5       $ g)zLHelper method to call Grok API for text generation (replaces pipeline calls))r   r   r   N )r   lenstrip)r   r   r   r   responses        r   _call_grok_for_generation"RAGAgent._call_grok_for_generationV   sJ     !#

 !F-3355>>##r   textc                 x    SSSSSSSSSSSS	S
.nUR                  5        H  u  p4UR                  X45      nM     U$ )z;Basic spelling correction for common OCR/misspelling issuesamericacolumbusbeforeculturesdestinysecret
philosophymystical	tradition)amercacolumbisbeforculurescultursr   clumbusr   secrect	philosopyr   r   )itemsreplace)r   r   correctionswrongrights        r   correct_spellingRAGAgent.correct_spellingd   sY      "!!! %"$
 (--/LE<<-D 0r   c                    Uc  gUnUR                  SS5      R                  SS5      n[        R                  " SSU5      n[        R                  " SSU5      n[        R                  " SSU5      n[        R                  " S	SU5      n[        R                  " S
SU5      n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  [        R                  -  S9n[        R                  " SSU[        R                  [        R                  -  S9n[        R                  " SSU5      n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SS U5      n[        R                  " SSU5      n[        R                  " SSU5      n[        R                  " S SU5      n[        R                  " S!SU5      n[        R                  " S"SU5      n[        R                  " S#SU5      n[        R                  " S$S%U5      nUR                  5       nU(       d  g[        U5      n[        UR                  5       5      nUS&:  a  X4S'-  :  a  UR                  5       $ U$ )(z>Clean common OCR artifacts while preserving legitimate contentr   u   —-   ¬ z\\<\^\^?z\^\^zK\d{2,}z\d{12,}z)\b\d{2}\s+\d{2}\s+\d{2}\s+\d{2}\s+\d{2}\bzorg/details/[^\s]+flagsz6Christopher\s+Columbus\s+and\s+the\s+participati[^\n]*zExpulsion of the Jews\.*z"Cornell University\s+Library[^\n]*z'Cornell\s+University[^\n]*Library[^\n]*z\bSAGE\s+ENDOWMENT\s+FUND[^\n]*z\bSAGE\s+ENDOWMENT[^\n]*z\bTHE\s+GIFT\s+OF[^\n]*zipmumyi;[^ ]*z The original ofthisbook.*?(?=\w)z<bought with the income from the sage endowment fund.*?(?=\w)z\bAS\bz\bbook\s*:\s*z\bstated earlier\s*,?z\bb^\bz$([A-Za-z])\s+([A-Za-z])\s+([A-Za-z])c                     U R                  S5      R                  5       (       a8  U R                  S5      U R                  S5      -   S-   U R                  S5      -   $ U R                  S5      $ )N      r      r   )groupislower)ms    r   <lambda>)RAGAgent.clean_ocr_text.<locals>.<lambda>   s    wxw~w~  @A  xB  xJ  xJ  xL  xLQRVWV]V]^_V`I`cfIfijipipqrisIs  J\  RS  RY  RY  Z[  R\  J\r   z(\w)\s+(\w)\.z\1\2.u	   [•%\^]+z_{2,}z
\b\d{4,}\b\s+\n+z([.!?])\s*([A-Z])\1 \2r   g      ?)r   resub
IGNORECASEDOTALLr   r   )r   r   original_textcleaned_lengthoriginal_lengths        r   clean_ocr_textRAGAgent.clean_ocr_textx   s1   < ||E3'//c: vvk2t,vvgr4(vvj"d+vvj"d+vvBBM vv+RR]]KvvOQSUYacananovv12t2==Qvv;RR]][vv@"dRTR_R_`vv8"d"--Xvv12t2==Qvv0"d"--Pvv&DF vv92t2==[][d[dKdevvUWY[_gigtgtwy  xA  xA  hA  B vviT*vv&DFvv.DNvviT? vv=  @\  ^b  cvv&$7 vvlC.vvhT* vvmR. vvfc4(vvfc4(vv*Hd;zz| Tm1134Q>d4J#J &&((r   c                    UR                  5       (       d  gSn[        UR                  5       5      nUS:  a  U[        US-  S5      -  n/ SQnUR	                  5       nU H  nXe;   d  M
  US-  nM     [        [
        R                  " SU5      5      nX'S-  -  nUR                  5       nSn	U H0  n
[        U
5      S	::  d  M  U
R                  5       (       d  M+  U	S
-  n	M2     U[        U	S5      -  nUR                  S5      nX+S-  -  n[        [
        R                  " SU5      5      nX,S-  -  n[
        R                  " SU5      n[        S U 5       5      nX.S-  -  n[        SU5      $ )z4Score the quality of text content (higher is better)        
   2          @)	zbook:k23cornelllibraryzoriginal ofthisbookzdr.z&christopher columbus andtheparticipatiipmumyizbought withtheincomez\d+皙?r   r   皙?      ?  z[bcdfghjklmnpqrstvwxyz]{4,}333333?[.!?]+c              3      #    U  H4  n[        UR                  5       R                  5       5      S :  d  M0  Sv   M6     g7f)   r   N)r   r   split.0ss     r   	<genexpr>1RAGAgent.score_content_quality.<locals>.<genexpr>   s,      TIqQWWY__=N9ORS9SIs   />	>)r   r   r   minlowerr   findallisalphacountsumrx   )r   r   quality_score
word_countocr_indicators
text_lower	indicatornumber_countwordsshort_word_penaltyworddouble_spacesgibberish_patterns	sentencescomplete_sentencess                  r   score_content_qualityRAGAgent.score_content_quality   sg   zz||  &
?Sb#66MfZZ\
'I&$ (
 2::fd34++ 

D4yA~$,,.."c)"  	/55 

4(,, !,JJ!WXc11 HHY-	  TI TTc113&&r   contextc                 p   UR                  S5      n/ nU GHW  nUR                  5       (       d  M  UR                  S5      n/ nU H  nUR                  5       (       d  M  UR                  S5      (       a  M2  UR                  S5      (       a  MJ  U R                  U5      nU(       d  Md  [	        UR                  5       5      S:  d  M  UR                  U5        M     U(       d  M  SR                  U5      n	U R                  U	5      n
U Vs/ s H2  owR                  S5      (       d  UR                  S5      (       d  M0  UPM4     nnSR                  X-   5      nUR                  X45        GMZ     UR                  S SS	9  US
S  VVs/ s H  u  pMUPM	     nnnSR                  U5      $ s  snf s  snnf )zDFilter and clean context chunks, prioritizing higher quality content
---

[Source:
Relevance:r   r   c                     U S   $ Nr    xs    r   r   1RAGAgent.filter_context_quality.<locals>.<lambda>  s    1r   TkeyreverseNrt   )	r   r   
startswithr   r   appendjoinr   sort)r   r   chunksscored_chunkschunklinescontent_lineslinecleaned_linecontent_textr   metadata_linesfiltered_chunkscorefiltered_chunkss                  r   filter_context_qualityRAGAgent.filter_context_quality   ss   y)E;;== KK%EM::<<
(C(CDOO\hLiLi#'#6#6t#<L#|L,>,>,@(AA(E%,,\:  }"xx6 $ : :< H 49!y54OOJ<W<W[_[j[jkw[x$5!y!%>+I!J$$n%DE+ 0 	~t<5B2A5FG5F\U55FG~~o.. "z Hs   /F- F-F2questionc                   ^ UR                  5       m/ SQn[        U4S jU 5       5      (       a  gST;   a  [        U4S jS 5       5      (       a  gTR                  S5      (       a  [        U4S jS	 5       5      (       a  gU R                  U5      (       a  g
[        U4S jS 5       5      (       a  g
[        U4S jS 5       5      (       a  g[        U4S jS 5       5      (       a  gg)z@Analyze the question to understand what kind of answer is needed)
z	which pdfzwhich documentz
which filezwhat other pdfzwhat other documentzwhat other filezwhat sourceszwhich sourceszlist the pdfszlist the documentsc              3   ,   >#    U  H	  oT;   v   M     g 7fNr  )r   phrasequestion_lowers     r   r   ,RAGAgent.analyze_question.<locals>.<genexpr>  s     L6KF'6K   source_lookupmentionc              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r   termr&  s     r   r   r'    s       /AR$~/ERr(  )pdfro   filesourcebooklistc              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r,  s     r   r   r'    s     4~X}PT^5KX}r(  )r.  ro   r/  r0  factualc              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r   r   r&  s     r   r   r'  $  s     a2`$~%2`r(  )rB   rV   rg   rW   r;   rC   c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r6  s     r   r   r'  &  s     V4UD'4Ur(  )explaindescribedefineexplanatoryc              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r6  s     r   r   r'  (  s     Y4XD'4Xr(  )compare
differencesimilarcomparativegeneral)r   r3   r  _detect_technical_query)r   r!  source_lookup_phrasesr&  s      @r   analyze_questionRAGAgent.analyze_question  s    !)!

 L6KLLL"&3  /AR  /A  ,A  ,A"$$V,,4~X}4~1~1~" ''11 a2`aaaV4UVVV Y4XYYY r   word1word2c                     U(       a  U(       d  g[         R                  " UR                  5       UR                  5       5      S-  $ )zBCalculate fuzzy similarity score between two words using rapidfuzzr   g      Y@)r   ratior   )r   rF  rG  s      r   fuzzy_match_scoreRAGAgent.fuzzy_match_score-  s,    Ezz%++-7%??r   c                    U(       d  / $ [         R                  " SSU5      n[         R                  " SU5      n/ nU HD  nU R                  UR	                  5       5      n[        U5      S:  d  M3  UR                  U5        MF     U$ )z,Split text into sentences with basic cleanupz(?<=\w)-(?=[A-Z]). z(?<=[.!?])\s+|\n   )r   r   r   r   r   r   r  )r   r   r   cleanedsentences        r   _split_into_sentencesRAGAgent._split_into_sentences3  ss    Ivv*D$7HH0$7	!H**8>>+;<H8}r!x( " r   max_sentences
max_lengthc                     U R                  U5      nU(       d  gSR                  USU 5      R                  5       n[        U5      U:  a  USU R	                  SS5      S   S-   nU$ )z4Create a short, clean preview snippet for UI displayr   r   Nr   r   ...)rQ  r  r   r   rsplit)r   r   rS  rT  r   previews         r   create_preview_textRAGAgent.create_preview_textA  sj    ..t4	((9^m45;;=w<*$kz*11#q9!<uDGr   limit_sentencesc                 r    U R                  U5      nSR                  USU 5      R                  5       nUSS $ )zNBuild a concise snippet from the cleaned context for summarisation or fallbackr   N   )rQ  r  r   )r   r   r[  r   snippets        r   consolidate_context_snippet$RAGAgent.consolidate_context_snippetL  s=    ..w7	((9%5o67==?t}r   max_sourcesc                 8   0 nU GHs  nUS   R                  SS5      n[        UR                  SS5      5      n[        UR                  SS5      5      nUS   R                  SS5      nUS   R                  S	/ 5      =(       d    / n	US   R                  S
5      n
US   R                  S5      nUR                  S5      nU R                  R                  U5      nU(       ak  UR                  S5      =(       d    UnUR                  S
5      (       a  US
   n
UR                  S	5      (       a  US	   n	UR                  S5      (       a  US   nU R                  R                  U5      nU(       aY  UR                  S5      =(       d    UnUR                  S
/ 5      nU(       a  Un
UR                  S	/ 5      =(       d    / nU(       a  Un	U
c  / n
U(       a  U R                  USS9nO<U(       a  UnO2U R	                  UR                  SS5      5      nU R                  U5      nU
(       a  U(       d  SR                  U
SS 5      nXS;   Ga  [        X5   S   U5      X5   S'   [        X5   S   U5      X5   S'   U(       a2  [        U5      [        X5   S   5      :  a  UX5   S'   U(       a  XU   S'   U
(       a%  X5   R                  S
/ 5      nU(       d
  U
SS X5   S
'   U	(       a%  X5   R                  S	/ 5      nU(       d
  U	SS X5   S	'   U(       a  X5   R                  S5      (       d  XU   S'   U(       a%  X5   R                  S5      (       d
  XU   S'   GMP  GMS  GMV  UUUUUU
(       a  U
SS O/ U	SS UUS.	X5'   GMv     [        UR                  5       S SS9nUSU $ )z?Aggregate search results by source and produce cleaned previewsmetadatar0  Unknownr  r   keyword_scoresummaryr   themes
key_pointsdocument_titlekeyword_snippetoverviewtitleNr   )rS  contentr   rX  r   r   )	r0  r  re  rX  rf  rh  rg  rl  rj  c                     U S   $ Nr  r  r	  s    r   r   3RAGAgent.prepare_source_summaries.<locals>.<lambda>      ajr   Tr  )getfloatr~   rY  r   r  rx   r   
setdefaultsortedvalues)r   search_resultsra  
aggregatedresultr0  r  re  rf  rg  rh  ri  rj  digestdigest_pointsdigest_themesrX  cleaned_contentexisting_pointsexisting_themes	summariess                        r   prepare_source_summaries!RAGAgent.prepare_source_summariesR  s   02
$FJ'++Hi@F&**Wc23E!&**_c"BCMZ(,,Y;GJ'++Hb9?RF
+//=J#J/334DEN$jj):;O**..v6F **Z0;G::l++!'!5J::h''#H-F::g&&%+G_N**..v6F **Z0;G &

< < !.J &

8R 8 >B *F!
22?RS2T!"&"5"5fjjB6O"P22?C'((:bq>2#.1*2DW2Mu.U
"7+69*:L_:]_l6m
"?3s7|c*2DY2O.PP4;J&y1&@O6*+<=&0&8&C&CLRT&UO*;Ebq>
*<8&0&8&C&CHb&QO*7=bqz
*84:#5#9#9)#D#D4;v&y1!**<*@*@*I*I2@v&w/ +J> %"%2&&4>*Ra.B$Raj+'6
&
"A %X :,,.4HRVW	+&&r   rw  limitc                    U(       d  / $ / n[        5       nU GHC  nUR                  S0 5      nUR                  S5      nU(       a  Xt;   a  M5  U R                  R                  U5      nU(       Ga  UU R                  XxR                  S5      =(       d    UR                  S5      5      UR                  S5      =(       d    UR                  S5      =(       d    SUR                  S5      =(       d    UR                  S5      =(       d    / UR                  S	5      =(       d    UR                  S	5      =(       d    / [	        UR                  S
S5      5      [	        UR                  SS5      5      S.n	OUR                  S5      =(       d$    U R                  UR                  SS5      SS 5      n
UU R                  XvR                  S5      5      U
UR                  S5      =(       d    / UR                  S	5      =(       d    / [	        UR                  S
S5      5      [	        UR                  SS5      5      S.n	UR                  U	5        UR                  U5        GMF     UR                  S SS9  USU $ )z<Collect distilled document digests relevant to the question.rc  r0  rl  ri  rk  rf  r   rh  rg  r  r   re  )r0  rl  rk  rh  rg  r  re  rm  N  c                     U S   U S   4$ )Nr  re  r  r	  s    r   r   5RAGAgent.collect_distilled_insights.<locals>.<lambda>  s    QwZ?1C$Dr   Tr  )	setrr  r~   _humanize_titlers  r   r  addr  )r   rw  r  insightsseenry  rc  r0  rz  insightrk  s              r   collect_distilled_insights#RAGAgent.collect_distilled_insights  s    I)+$Fzz*b1H\\(+FV^**..v6F$!11&**W:M:oQYQ]Q]^nQop &

: 6 W(,,y:Q WUW"(**\":"^hll<>X"^\^$jj2Rhll86LRPR"6::gs#;<%*6::os+K%L $<<	2jd6I6I&**U^`bJcdhehJi6j$!11&,,GW:XY ("*,,|"<"B&ll84:"6::gs#;<%*6::os+K%L OOG$HHV= %@ 	DdSr   r  question_typec                    U(       d  g[        [        R                  " SUR                  5       5      5      nU R	                  U5      nU(       d  Un/ nU GH  n[        5       nUR                  S/ 5       HH  n	[        R                  " SU	R                  5       5      n
UR                  U R	                  U
5      5        MJ     [        5       nUR                  S/ 5       HH  n[        R                  " SUR                  5       5      nUR                  U R	                  U5      5        MJ     UR                  X-  5      n[        U5      [        S[        U5      5      -  nSnUR                  SS5      R                  5       nUR                  5       nSU;   d  S	U;   a  S	U;   d  S
U;   a  SnODSU;   d  SU;   a  SU;   d  SU;   a  SnO)SU;   d  SU;   a  SU;   d  SU;   a  SnOSU;   a  SU;   a  SnUS   S-  UR                  SS5      S-  -   U-   U-   nUR                  UU45        GM     UR                  S SS9  USS  VVs/ s H  u  nowPM	     snn=(       d    USS n/ n[        5       n/ nU GH4  nUR                  S5      =(       d(    [        US   5      R                  R                  SS5      nUR                  U5        U R                  XW5      nU GH  nU(       a  [        UR!                  5       5      S :  a  M*  U R#                  U5      nU(       d  MD  [        R$                  " S!SUR                  5       R!                  5       5      n[        R$                  " S"SU5      nUR'                  5        Vs/ s H  n[        U5      S:  d  M  UPM     nn[        U5      S#:  a  M  S$n U H  n![        U!R'                  5        Vs/ s H  n[        U5      S:  d  M  UPM     sn5      n"[        U5      n#[        U#5      S%:X  d  [        U"5      S%:X  a  Mi  [        U#R                  U"5      5      n[        U#R)                  U"5      5      n$U$S%:  d  M  UU$-  S:  d  M  Sn   O   U (       a  GM  UR                  U5        UR+                  SR-                  U5      5        GM     GM7     U(       d  gU R/                  UUU5      n%/ n&U%(       a3  U R1                  U%5      n'U&R                  U'5        U&R                  S5        U(       a  / n([3        USS& 5       H]  u  n)nU(R                  U5        [        U(5      S:  d  M(  SR-                  U(5      n*U&R                  U*5        U&R                  S5        / n(M_     U((       a   U&R                  SR-                  U(5      5        S'R-                  S( U& 5       5      R!                  5       n+U R5                  U+5      n+[        U+5      S):  a  gU R7                  U+5      (       a  gU R9                  U5      n,U,(       a  S*S+R-                  U,SS 5      -   n-U+U--   n+U+$ s  snnf s  snf s  snf ),z=Generate a synthesized answer directly from document digests.r   \b\w+\brg  rh  r   r   r0  
barry fellfellbarryr   	blavatskysecret doctrinesecret_doctrine
manly hallhallmanlyr   r  ffffff?re  r   c                     U S   $ Nr   r  r	  s    r   r   9RAGAgent.answer_from_distilled_insights.<locals>.<lambda>  s    !A$r   Tr  Nr   r   rl  _r   rN  [^\w\s]r   r   Fr      r  c              3   R   #    U  H  oR                  5       (       d  M  Uv   M     g 7fr$  r   r   r  s     r   r   :RAGAgent.answer_from_distilled_insights.<locals>.<genexpr>b  s     ILDJJL44L   '	'<   z

Sources: , )r  r   r   r   _filter_stop_wordsrr  updateintersectionr   rx   r  r  r   stemr   _extract_relevant_sentencesr   _clean_sentencer   r   unionr  r  _build_intro_from_insights_clean_intro	enumerate_format_final_answeris_generic_output_dedupe_preserve).r   r!  r  r  question_words_rawquestion_wordsscoredr  theme_wordsthemetheme_tokenskey_point_wordspoint
key_tokensoverlap	relevanceauthor_match_boostsource_namer&  combined_scorer  selectedall_sentencesseen_sentencessource_titlesrl  r   rP  rO  
normalizedwr   is_duplicate	seen_norm
seen_wordscurrent_wordstotal_uniqueintroanswer_partsintro_cleancurrent_parai	para_textanswerunique_sourcessources_lines.                                                 r   answer_from_distilled_insights'RAGAgent.answer_from_distilled_insights  s     J8H!IJ001CD/N57G%K Xr2!zz*ekkmD""4#:#:<#HI 3 "eO \26ZZ
EKKMB
&&t'>'>z'JK 7 %11+2OPGGs1c..A'BBI "%!++h399;K%^^-N ~->1I[(G{,B),&.2C~2U+-1Bk1Q),&/6^3K[(G{,B),&~-,),&  3&OS1C78 ##  MM>734O  R 	5.4Raj9j
7Gj9IXbq\ GKK(ZD1B,C,H,H,P,PQTVY,ZE  '88QI%3x~~'7#82#= ..x8  VVJGMMO4I4I4KL
VVFC<
 %/$4$4$6E$6q#a&1*$6Eu:>  %!/I!$1B%Q1BAc!fqja1B%Q!RJ$'JM=)Q.#j/Q2F !-"<"<Z"HIG#&}':'::'F#GL#a'Gl,BS,H'+ "0 $|!((1"&&sxx7I &  V  //(MR ++E2K,# L(r):;8##H- |$) # 6I ''	2 ''+#%L  < ##CHH\$:; ILIIOOQ **62v;!!&)) ..}=*TYY~bq7I-JJLl*FI :4 F &Rs   X4;X:X:	X? X?c                    U(       d  gUS:X  a  SnOUS:X  a  SnOSn/ nU H;  nUR                  S/ 5       H"  nUR                  UR                  5       5        M$     M=     U R                  U5      nU(       a  SR	                  US	S
 5      nU U SU S3$ U Vs/ s H+  ofR                  S5      (       d  M  UR                  S5      PM-     n	nU R                  U	5      n	U	(       a  SR	                  U	S	S 5      OSnU U SU S3$ s  snf )zCCompose an introductory sentence summarizing the combined insights.r   r4  z&According to the distilled documents, r;  The documents explain that zThe sources indicate that rg  r  Nrt   u2    feature prominently in discussions related to “u   ”.rl  r   zthe referenced sourcesu!    provide accounts relevant to “)rr  r  rl  r  r  )
r   r!  r  r  leadtheme_termsr  r  joined_entitiesentitiess
             r   r  #RAGAgent._build_intro_from_insightst  s   I%;Dm+0D/DG Xr2""5;;=1 3   ++K8"iiBQ8OVO,,^_g^hhlmm<D]HT[H\,G,HH],,X6H9Adii!5G_OVO,,MhZW[\\ ^s   D7DNc                 ,  ^^^ UR                  5       n1 Skn[        R                  " SU5       Vs/ s H  n[        U5      S:  d  M  Xe;  d  M  UPM     nnU(       d  g[	        5       nU HT  nUR                  U5        UR                  S5      (       a  UR                  USS 5        M@  UR                  US-   5        MV     / m[	        5       mS[        S	[        S
[        4UU4S jjn	U(       a  U H  n
U
R                  S0 5      R                  SS5      n[        U
R                  S	S5      5      nU R                  U
R                  SS5      =(       d    S5      nUR                  5       m[        U4S jU 5       5      (       d  M  U R                  USSS9nU	" XU5        M     T(       Gd8  U(       Ga0  UR                  S5      nU GH  nSU;  a  M  UR                  S5      n[        S U 5       S5      nUR                  SS5      R                  SS5      R!                  5       =(       d    Sn [        S U 5       S5      n[        UR                  SS5      S   R!                  5       5      nSR%                  S U 5       5      nU R                  U5      nUR                  5       m[        U4S jU 5       5      (       d  M  U R                  USSS9nU	" XU5        GM     T(       d  gTR'                  S SS 9  S!R%                  [)        [	        U5      5      5      nS"U S3/n[+        TSS#9 H5  u  nnUS
   (       a  S$US
    3OSnUR-                  U S%US    U 35        M7     SR%                  U5      $ s  snf ! ["         a    Sn GNf = f)&zBGenerate answer listing which documents mention the queried topic.>%   r&   r3   r.  r1  rG   elser/  r2  pdfsshowtalkrV   r[   bookscoverfileslistsr`   showstalkstopicrh   coversr0  anotherdiscussincluder*  sourcesro   includesmentions	discussesrr   	mentioned	reference
referencesr  r   r   r   Nr0  r  r^  c                    > U T;   a  g TR                  U 5        TR                  U UU(       a  UR                  5       OSS.5        g )Nr   )r0  r  r^  )r  r  r   )r0  r  r^  matchesseen_sourcess      r   	add_match7RAGAgent.create_source_lookup_answer.<locals>.add_match  s@    %V$NN .57==?2 r   rc  rd  r   rm  c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r   kwcontent_lowers     r   r   7RAGAgent.create_source_lookup_answer.<locals>.<genexpr>       G5Fr]*5Fr(     rS  rT  r  r  r  c              3   T   #    U  H  oR                  S 5      (       d  M  Uv   M      g7f)r  Nr  r  s     r   r   r    s     #ZUTooj>YDDU   (	(]c              3   T   #    U  H  oR                  S 5      (       d  M  Uv   M      g7f)r  Nr  r  s     r   r   r    s     &_Q]A^ttr	  zRelevance: 0:r   c              3      #    U  H?  o(       d  M  UR                  S 5      (       a  M#  UR                  S5      (       a  M;  Uv   MA     g7f)[r  Nr  r  s     r   r   r    s>       $GUTdD4??[^K_Dhlhwhw  yE  iFDDUs   
A	A	A	 	A	c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r   s     r   r   r    r  r(  c                     U S   $ ro  r  r	  s    r   r   6RAGAgent.create_source_lookup_answer.<locals>.<lambda>  s    1W:r   Tr  r  z The following documents mention startu    — rM  )r   r   r   r   r  r  endswithstrrs  rr  r   r3   rY  r   nextr   r   r   r  r  ru  r  r  )r   r!  r   rw  r&  
stop_wordsr   keywordsexpanded_keywordsr  ry  r0  r  rm  r^  r  r  r  source_line
score_linetopic_phraseidxmatchsnippet_partr  r  r  s                           @@@r   create_source_lookup_answer$RAGAgent.create_source_lookup_answer  sn    ")

  ZZ
NC
CT4y1} !%!7 C 	 

   ED!!$'}}S!!!%%d3Bi0!%%dSj1  )+u	c 	% 	# 	 	 (J377)Lfjj#67--fjjB.G.M2N 'G5FGGG"66wa\_6`GfW5 ) 7]]9-FU*D)"#ZU#Z\^_$,,Z<DDS"MSSUbYb !%&_&_ao!pJ!*"2"23":1"="C"C"EFE ))  $GU  $G  G--g6 'G5FGGG"66wa\_6`GfW5#  & -t<yyH!67.|nA>
 $G15JC9>y9IU5#3"45rLLLC55?"3L>BC 6 yy]
l !  E s#   M>M>M>4?NNNc                    UR                  5       n[        [        R                  " SU5      5      nU Vs1 s H  n[	        U5      S:  d  M  UiM     nn1 SknU Vs1 s H  oUU;  d  M
  UiM     nnU(       d#  U Vs1 s H  n[	        U5      S:  d  M  UiM     nnU(       d  Un[        U5      n/ SQ/ SQ/ SQ/ SQ/ SQ/ S	Q/ S
Q/ SQ/ SQ/ SQS.
n	U H  n[	        U5      S:  d  M  UR                  USS 5        UR                  S5      (       a  UR                  USS 5        OUR                  US-   5        US;   a  UR                  / SQ5        O3US;   a  UR                  / SQ5        OUS;   a  UR                  / SQ5        XY;   d  M  UR                  X   5        M     UR                  S5      n
/ nU
 GH  nUR                  5       (       d  M  SnSU;   aF  SU;   a@   UR                  S5      S   R                  S5      S   R                  5       n[        U5      nUR                  S 5      nSn[        U5       H7  u  nnUR                  5       (       d  M  UR                  S5      (       a  M5  Un  O   S R!                  UUS 5      n[        R                  " S!U5      nU GH  nUR                  5       n[	        U5      S":  a  M%  U R#                  U5      n[	        U5      S":  a  MG  UR                  5       n[        [        R                  " SU5      5      nSn/ nUR%                  U5      nU(       a"  U[	        U5      S"-  -  nUR'                  U5        U H  nU Hy  n[	        U5      S:  d  M  [	        U5      S:  d  M%  U R)                  UU5      nUS#:  a  UUS$-  -  nUR+                  U5        MX  US%:  d  M`  UUS&-  -  nUR+                  U5        M{     M     UR%                  U5      n U (       a"  U[	        U 5      S'-  -  nUR'                  U 5        UUS-  -  n[	        [        U5      5      n!U!S:  a  UU!S-  -  nUS:  d  GM  UR+                  UUU45        GM     GM     UR-                  S( S)S*9  USS+  VV"s/ s H	  u  n  n"UPM     n#nn"U#(       Gd^  / n$U
 H^  nSn%SU;   a@   UR                  S5      S   R                  S5      S   R                  5       n[        U5      n%U$R+                  UU%45        M`     U$R-                  S, S)S*9  U$SS  H  u  nn"UR                  S 5      nSn[        U5       H7  u  nnUR                  5       (       d  M  UR                  S5      (       a  M5  Un  O   S R!                  UUS 5      n[        R                  " S!U5      nU V&s/ s H2  n&[	        U&R                  5       5      S-:  d  M"  U&R                  5       PM4     sn&SS n'U#R'                  U'5        M     S.R!                  U#SS& 5      n(U R#                  U(5      $ s  snf s  snf s  snf ! [        [        [        4 a  nSn SnAGN1SnAff = fs  sn"nf !   Sn% GNu= fs  sn&f )/zQExtract the most relevant information from context with improved keyword matchingr  r   >&   r2   r7   r9   r:   r@   rB   rF   rG   rJ   rK   rL   ontorS   rT   uponrU   rV   rW   rY   rZ   r[   afterr]   r^   r`   ra   rb   rc   rg   rh   rj   r   pleaserk   ro   rp   rq   rr   rt   )	continent	territorynewworld)civilizationpeopletribenation)r,  tribesnationsancient)r/  r0  r   r1  )serpentserpentssnakesnakes)r2  r3  r4  r5  wisdom)voyagers
navigatorsexpeditions)finance	financingcapitalbacking)commerceexchangenetworkroutes)
r   americasculturepeoplesr,  naganagas	explorersfundingtrader   Nr   r  )r   rB  )landr(  r)  r*  )rC  r   )r+  societyr,  r-  r.  )r   r   pre)priorr1  early
indigenousnativer  r   r  r  r   r
  r   r  r      皙?r   333333?r   r  c                     U S   $ r  r  r	  s    r   r   +RAGAgent.extract_key_info.<locals>.<lambda>y  s    AaDr   Tr     c                     U S   $ r  r  r	  s    r   r   rU    s    !A$r   rN  r   )r   r  r   r   r   r  r  r  r   r   rs  
IndexError
ValueErrorAttributeErrorr  r  r  r   r  extendrJ  r  r  ))r   r   r!  r&  raw_question_wordsr   r  r  expanded_wordssynonym_mapr  scored_sentencesr  
base_score
score_partr   r  content_startr  r  rm  r   rP  sentence_cleansentence_lowersentence_wordsrelevance_scorematched_termsexact_matchesq_words_word
similaritysynonym_matchesunique_matchesr  top_sentenceschunks_with_scoresr  r   chunk_sentencescombineds)                                            r   extract_key_infoRAGAgent.extract_key_info  s,    ") J!GH/AS/AtSYQR]d/AS

 ,>X+=4ZAW$+=X/AS/AtSYQR]d/ANS/N ^,ABDAB>IBEB
 #D4y1}""48,==%%"&&tCRy1"&&tcz2 22"))*OP44"))*bc::"))*_`&"))+*;<' #, y)E;;== JU"|u'<%!&\!:1!=!C!CC!H!K!Q!Q!SJ!&z!2J KK%EM$U+4::<<
(C(C$%M ,
 iimn 56GG4I%!)!1~&+ "&!4!4^!D~&+!/!5!5!7!$RZZ
N%K!L"# " !/ ; ;N K #s='9B'>>O!((7 -F"0v;?s6{Q)-)?)?)OJ)C/ /:? B - 4 4V <!+c!1 /:> A - 4 4V < #1 - #1"="=n"M"#s?';a'??O!((9  :>1 "%S%7!8!A%#~'99O"Q&$++^_j,YZc &3 Z 	.$?8H!8LM8Lnh18LM !#5($%*[[%>q%A%G%G%LQ%O%U%U%W
 %j 1 #))5%.9   ###E.r2qD) !(/GAtzz||DOOJ,G,G()  0 ))E-.$9:HHY8	6?"Wi3qwwy>TVCV91779i"WXZYZ"[$$_5 3 88M"1-.""8,,G T YSj #J? %!$J%H N$ # #XsY   X'X'	X,)X,<X1X1#?X6	Y4?Y!.!Y+Y+6YYY!Y(c                 t   [         R                  " SU5      nU Vs/ s H2  n[        UR                  5       5      S:  d  M"  UR                  5       PM4     snSS nU(       d  gSR	                  U5      nUR                  S5      (       d  US-  nU(       a  US   R                  5       US	S -   nU$ SnU$ s  snf )
z3Format extracted information into a coherent answerr   r   Nr   r   rM  .r   r   )r   r   r   r   r  r  upper)r   extracted_infor  r   r   meaningful_sentencesr  s          r   format_extracted_answer RAGAgent.format_extracted_answer  s     HHY7	3<T9aAGGIQS@S		9TUWVWX# /0s##cMF 4:"VABZ/ @B  Us   !B5B5c                     UR                  5       nSU;   a  gSU;   a  gSU;   d  SU;   d  SU;   a  gS	U;   d  S
U;   a  gU R                  $ )z-Provide fallback answers for common questionszsecret destiny of americazThe Secret Destiny of America is a book by Manly P. Hall that explores the esoteric and philosophical foundations of American democracy, suggesting that America has a special spiritual destiny in world affairs, rooted in hermetic and mystical traditions.rE  zThe Nagas were semi-divine beings in Hindu and Buddhist mythology, often depicted as serpents or having human upper bodies with serpent lower bodies. They were considered wise beings associated with water, wisdom, and sometimes royalty.r  r  zp halla  Manly P. Hall was an American mystic, author, and philosopher known for his extensive research into esoteric traditions, alchemy, and mysticism. He authored numerous books including 'The Secret Destiny of America' and was a prominent figure in the study of hermetic philosophy.r  r  zThe Secret Doctrine is a book by Helena Petrovna Blavatsky, published in 1888. It outlines her teachings on the origins of humanity, the nature of the universe, and esoteric philosophy, drawing from various religious and mystical traditions.)r   GENERIC_FALLBACK_MESSAGE)r   r!  r&  s      r   get_fallback_answerRAGAgent.get_fallback_answer  sn     ") '.8 T^# B&&N*BhR`F` k.0K>4Q G 000r   c                   ^ U(       d  U R                  U5      $ UR                  S5      n/ n/ nUSS  H  nSU;   d  M  UR                  S5      S   nUR                  S5      S   R                  SS	5      nUR                  U5        UR                  S5      n	U	 V
s/ s HB  oR	                  5       (       d  M  U
R                  S
5      (       a  M1  U R                  U
5      PMD     nn
U(       d  M  U R                  SR                  U5      SSS9nU(       d  M  UR                  U SU 35        M     UR                  5       mU R                  U5      n[        U4S jS 5       5      (       a<  [        S U 5       5      (       a%  SU(       a  US   OS-   S-   U(       a  US   -   $ S-   $ [        U4S jS 5       5      (       a<  [        S U 5       5      (       a%  SU(       a  US   OS-   S-   U(       a  US   -   $ S-   $ [        U4S jS 5       5      (       a<  [        S U 5       5      (       a%  SU(       a  US   OS-   S-   U(       a  US   -   $ S -   $ [        U4S! jS" 5       5      (       a<  [        S# U 5       5      (       a%  S$U(       a  US   OS-   S-   U(       a  US   -   $ S%-   $ U(       a  U(       a  S&US   -   S-   US   -   S'-   $ S(U-   S)-   $ s  sn
f )*zHCreate intelligent, agentic fallback using available context informationr  Nr   r  r  r   r
  	[Source: r   r  r   r      r  : c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r,  s     r   r   6RAGAgent.get_context_aware_fallback.<locals>.<genexpr>  s     p2o$~%2or(  )rC  r,  r+  rP  rO  c              3      #    U  HL  nS UR                  5       ;   =(       d-    SUR                  5       ;   =(       d    SUR                  5       ;   v   MN     g7f)r   settlerr  Nr   r   s     r   r   r    sE     ldk_`9	)ZY!'')-CZvQRQXQXQZGZZdks   AAzQI found relevant information about pre-Columbian cultures in the Americas. From 'zthe documentsz': zdThe documents discuss ancient settlements and civilizations in the Americas before European contact.c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r,  s     r   r   r    s     ~2}$~%2}r(  )r   christophervoyage	discoveryspanish
portuguesec              3   H   #    U  H  nS UR                  5       ;   v   M     g7fr   Nr  r   s     r   r   r         <Gq:*G    "zXThe documents contain information about Christopher Columbus and his expeditions. From 'zSColumbus's voyages and the historical context of Spanish exploration are discussed.c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r,  s     r   r   r    s     X2W$~%2Wr(  )jewjewishhebrewisraelc              3   H   #    U  H  nS UR                  5       ;   v   M     g7fr  r  r   s     r   r   r    r  r  zoThe documents explore the participation of Jewish individuals and communities in Columbus's expeditions. From 'zNJewish involvement in Spanish and Portuguese maritime discoveries is examined.c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r,  s     r   r   r    s     q2p$~%2pr(  )r   r   r  r  esotericr   c              3   |   #    U  H2  nS UR                  5       ;   =(       d    SUR                  5       ;   v   M4     g7f)r  r   Nr  r   s     r   r   r    s.     R'Q6QWWY&@)qwwy*@@'s   :<zJThe documents include esoteric interpretations of American history. From 'zHManly P. Hall's work on the mystical foundations of America is featured.zrWhile I couldn't extract a direct answer to your specific question, the documents provide relevant context. From 'z^ This information may help address aspects of your query about historical and esoteric topics.zaI analyzed the available documents but couldn't find a definitive answer to your question about 'z'. The sources appear to focus on historical and esoteric topics. You might try rephrasing your question or asking about specific aspects covered in the documents.)r}  r   r   r  r   r  r   rY  r  r   rD  r3   )r   r   r!  r  r  top_snippetsr  r  r  r  r  r  rX  r  r&  s                 @r   get_context_aware_fallback#RAGAgent.get_context_aware_fallback  s   ++H55 y)BQZEU"#kk$/2)//4Q7??RP{+ D)GL zutPZPZP\!:eietetuxey!:!4!4T!:u z ="66sxx7N^_lo6pGw$++{m2gY,GH   ")--h7 p2opppldklllj  }Dnuvwnx  JY  Z  ]b  b  yE  fr  st  fu  r  r  Kq  r  r~2}~~~<G<<<q  DKu|}~u  Q`  a  di  i  @L  my  z{  m|  h  h  Rg  h  hX2WXXX<G<<< I  [b  MT  UV  MW  hw  x  {@  @  Wc  DP  QR  DS  z  z  iy  z  zq2pqqqR'RRRcu|gnopgq  CR  S  V[  [  r~  _k  lm  _n  O  O  DN  O  O | H  KR  ST  KU  U  X]  ]  `l  mn  `o  o  rR  R  R su}}  Af  f  	fA !{s   K9KKspecific_fallbackdistilled_insightsc           	         U R                  U5      nU R                  U5      nUc  U R                  U5      nUSL=(       a    X@R                  :g  nU R	                  Xv5      n	U	(       a  U R                  U	5      OSn
U(       a  UR                  5       (       d  U$ U
(       aO  [        U
R                  5       5      S:  a2  U R                  U
5      nUS:  a  U R                  XjU5      nU(       a  U$ U R                  XjXs5      nU(       a  U$ U R                  UU
UUUU=(       d    / 5      nU(       a  XR                  :w  a  U$ U R                  X&5      nXR                  :X  a	  U(       a  U$ U$ )z3Multi-step reasoning process for generating answersNr   r   rS  )r   r  r}  r|  rr  r   r   r   r   create_answer_from_extractionsummarize_from_contextgenerate_with_reflectionr  )r   r!  r   r  r  r  corrected_questioncleaned_contextuse_specific_fallbackkey_infokey_info_cleanr   formatted_answersynthesized	generated
contextuals                   r   reason_step_by_stepRAGAgent.reason_step_by_step  sw    "228< 55g> $ $ 8 89K LT)`.?C`C`.` 	
 ((M:B,,X6 !)>)>)@)@$$ c."6"6"89B> 66~FMs"#'#E#EFXjw#x #++ 112DVeu 11$"
	 &C&CC 44WQ
666;P$$r   r  r  full_contextoriginal_contextc                    US:X  a  SnOUS:X  a  SnOUS:X  a  SnOSn[        U5      S:  a  US	S OUnU(       a  US	S
 OSnUR                  UUUS9n	 U R                  U	[        [        S5      [        [        S5      S9n
U
(       d  U	n
S	nSU
;   a#  U
R                  S5      S   R                  5       nOSU
;   a#  U
R                  S5      S   R                  5       nOSU
;   a#  U
R                  S5      S   R                  5       nOfSU
;   a#  U
R                  S5      S   R                  5       nO=[        U	5      n[        U
5      U:  a  XS	 R                  5       nOU
R                  5       nX;   a   UR                  U	S5      R                  5       nUR                  S5      n[        U5      S:  aC  US   R                  5       US   R                  5       :X  a  SR                  US   /USS	 -   5      nU(       a  U R                  X5      OS	nU(       a  [        UR                  5       5      S:  a  U R                  XQ5      $ UR                  5       nUR                  S5      (       d-  UR                  US	S 5      S:  d  U R!                  U5      S:  a  U R                  XQ5      $ U$ ! ["         a)  n[%        SU 35        U R                  X15      s S	nA$ S	nAff = f) z@Generate answer using the language model with improved promptingr)  a  You are an archivist assisting with document retrieval. Use only the provided context to identify which documents mention the requested topic. Present the results as a concise numbered list with the document title followed by a short supporting snippet.

QUESTION: {question}

KEY CONTEXT: {key_info}

FULL CONTEXT: {context}

INSTRUCTIONS:
- Only include documents that clearly mention the topic
- Quote or summarise the relevant mention in one short phrase
- If the topic is not mentioned, state that it is not found

RESULTS:r4  a_  You are an intelligent research assistant analyzing documents. Answer this factual question directly and precisely, using only information from the provided context.

QUESTION: {question}

KEY CONTEXT: {key_info}

FULL CONTEXT: {context}

INSTRUCTIONS:
- Answer factually and concisely
- Use evidence from the documents
- If information is unclear, say so rather than speculate
- Structure your answer clearly
- For technical questions (pH, temperature, measurements), extract specific numbers/ranges from the context
- Do NOT provide generic information if the specific answer isn't in the context

ANSWER:r;  a  You are a knowledgeable analyst explaining historical and esoteric topics from primary sources. Provide a clear, evidence-based explanation.

QUESTION: {question}

KEY CONTEXT: {key_info}

FULL CONTEXT: {context}

INSTRUCTIONS:
- Explain concepts using document evidence
- Break down complex ideas step by step
- Maintain historical accuracy
- Be comprehensive but concise

EXPLANATION:a  You are a thoughtful researcher synthesizing information from historical documents to address complex questions.

QUESTION: {question}

KEY CONTEXT: {key_info}

FULL CONTEXT: {context}

INSTRUCTIONS:
- Synthesize information from multiple sources when relevant
- Provide balanced, evidence-based insights
- Acknowledge uncertainties in the historical record
- Connect ideas meaningfully

RESPONSE:i  Nr]  z9No specific key information extracted from the documents.)r!  r  r      r   r   r   zAnswer:r  zANSWER:zEXPLANATION:z	RESPONSE:r   ru  r   r   r   rM  rN   based on the available documentsr         ?zError in model generation: )r   formatr   r   r
   rx   r   r   r   r   r  post_process_answerr  r   r  r   r   r   r   )r   r  r  r  r  r  prompt_templatecontext_summarykey_info_summaryr   generated_textr  
prompt_lenr   answer_lowerr   s                   r   generate_with_modelRAGAgent.generate_with_model5  s    O+O i'O" m+O O" 25\1BT1I,u-|-58DS>;v !'''%# ( 
;	U!;;~s3S1 < N
 "!' F N*'--i8<BBDn,'--i8<BBD>1'--n=bAGGI.'--k:2>DDF ![
~&3+K8>>@F+113F 399; S)I9~!Q<%%'9Q<+=+=+??!YY	!~	!"'EFF IOT--fDTXF S025667G\\ "<<>L''(JKK""<#459**62S8667G\\M 	U/s3422<TT	Us,   G/J* A#J* (J* *
K4KKKr  c                    U R                  U5      nSR                  S U R                  U R                  * S  5       5      =(       d    SnUSS n	U(       a  USS OSn
U R	                  U5      nSU S	U(       a  UOS
 SU
 SU SU	 SU(       a  UOS S3n U R                  US[        [        S5      S9nU(       a]  U R                  R                  U5        [        U R                  5      U R                  :  a  U R                  U R                  * S U l        U R                  UUUU
UUU5      n U R                  U[        [        S5      [        [        S5      S9nU R!                  U5      u  nnU(       a  U R#                  UR%                  5       5      SS nU(       a`  U R                  R                  SU 35        [        U R                  5      U R                  :  a  U R                  U R                  * S U l        U(       d  U R                  XX4U5      $ U R'                  UU5      n[        U5      S:  a  U R                  XX4U5      $ U$ ! [         a+  n[        SU 35        U R                  XX4U5      s SnA$ SnAff = f! [         a+  n[        SU 35        U R                  XX4U5      s SnA$ SnAff = f)z?Two-stage reflective generation to simulate internal reasoning.r  c              3   ,   #    U  H
  nS U 3v   M     g7f- Nr  r   notes     r   r   4RAGAgent.generate_with_reflection.<locals>.<genexpr>       "g;f4Rv;;f   NzNone recorded.  X  z$No specific key facts extracted yet.zaYou are preparing internal research notes. Think carefully before answering the user.

QUESTION: 

DISTILLED INSIGHTS:
None provided.

KEY INFO:
z

SHARED MEMORY:
z

CONTEXT SNAPSHOT:


ENTITY GRAPH HINTS:
z0No entity relationships surfaced for this query.z

Write concise bullet-style analysis notes capturing the most relevant facts, names, places, and time periods. Do not answer the question yet. Focus on evidence and uncertainty.   r   r  z%Warning: reflection analysis failed:    皙?  zTrace: (   z'Warning: reflection generation failed: )_format_insight_sectionr  r|   r}   _format_entity_graph_hintsr   rx   r   r   r   r  r  r   _build_reflective_promptr   r
    _extract_final_answer_from_reactr   r   r  )r   r  r  r  r  r  r  insight_sectionmemory_sectionr  key_info_segmententity_hint_sectionanalysis_promptanalysis_textr   final_promptreact_outputfinal_answer_rawreact_tracememor  s                        r   r  !RAGAgent.generate_with_reflection  s    667IJ"g4;Q;QSWSdSdRdRe;f"gg{k{)%40-58DS>;a"==>PQ
  $)9 : ;          ,1c d eq#t*		| ::S1 ; M ""))-84))*T->->>)-)?)?ARAR@R@S)T&44
	|99~s3S1 : L
 -1,Q,QR^,_)k**;+<+<+>?E**11GD62BC4112T5F5FF151G1GIZIZHZH[1\.#//0Bon~--.>NF6{R//0Bon~MQ  	|9!=>++,>/jz{{	|R  	|;A3?@++,>/jz{{	|sI   I 2C*J 3J J 
J J=JJ
K  J;5K ;K c                    U(       a  U R                  U5      O/ nU(       d!  U R                  USS9nU R                  U5      nU(       d  g/ n[        5       nU HK  n	U	R                  5       n
X;  a"  UR	                  U	5        UR                  U
5        [        U5      S:  d  MK    O   U(       d  g/ n[        U5       Hy  u  pU	R                  5       n	U	R                  S5      (       d  U	S-  n	US:X  a  UR	                  U	5        MI  US:X  a  UR	                  SU	 35        Me  UR	                  S	U	 35        M{     S
R                  U5      nU R                  X5      n[        U5      S:  a  gU$ )zICreate a deterministic summary-style answer when direct extraction fails.rV  )r[  r   r   ru  r   r   zAdditionally, zFurthermore, r   r  )rQ  r_  r  r   r  r  r   r  r   r  r  r  )r   r!  r  r  r  r   r^  unique_sentencesr  rP  r  r  r  r  s                 r   r  RAGAgent.summarize_from_context  sX   <DD..x8"	66XY6ZG227;Iu!H!)J% ''1$#$) "  &'78MC~~'H$$S))Cax##H-##nXJ$?@##mH:$>? 9 ,'))&@v;r   c                 &   U R                  U5      nU R                  U5      nU R                  XT5      nU(       aG  [        UR	                  5       5      S:  a*  U R                  XFU5      nU(       a  [        U5      S:  a  U$ U R                  X$5      $ )zOCreate answer directly from extracted content without language model generationrN     )r   r  rr  r   r   r  r  )r   r!  r   r  r  r  r  direct_answers           r   create_direct_answerRAGAgent.create_direct_answerH  s     "228< 55g> ((M HNN,-2 >>?Q]jkM]!3b!8$$ ..wKKr   rw  c                    U R                  U5      n[        R                  " SUR                  5       5      n/ nU GH	  nUR                  5       n[	        U5      S:  d  M%  [        R
                  " SU5      (       a  MC  [        R
                  " SU[        R                  5      (       a  Mp  U R                  U5      S:  d  M  [        R                  " SSU5      n[        R                  " SSU5      n[        R                  " S	SU5      n[        R                  " S
SU5      n[	        U5      S:  d  M  UR                  U5        GM     USS nU(       d  gUS:X  a#  SUS    3n[	        U5      S:  a  USUS    3-  nOKUS:X  a#  SUS    3n[	        U5      S:  a  USUS    3-  nO"SUS    3n[	        U5      S:  a  USUS    3-  nUR                  5       nUR                  S5      (       d  US-  n[	        [        R                  " SU5      5      n	[	        U5      S:  d  U	S:  d  U R                  U5      S:  a  gU$ )zYCreate a coherent, agentic answer from extracted information with aggressive OCR cleaningr      z^[^\w]*\d+[^\w]*$zS^.*(?:org/details|K\d+|CHRISTOPHER|cornell|ibrary|original ofthisbook|dr\.|\^\^).*$r   r   r   z[^A-Za-z0-9,;:'\"()\-\s]z\s{2,}z^[^\w]+|[^\w]+$r   Nr   r4  zAccording to the documents, r   r   z Additionally, r;  r  z Furthermore, zBased on the source material, z, This is further supported by evidence that ru  z(\b\d{4,}\b|org/|http|www\.|\^\^|K\d+|<|>r   r   r  )r   r   r   r   r   r  r   r   r   r  r  r   )
r   r!  rw  r  cleaned_infor   rx  sentr  artifact_counts
             r   r  &RAGAgent.create_answer_from_extraction]  sN    **>: HHY(:(:(<=	!D::<DD	BHH1488HHsuy{}  |I  |I  J  J**4036 vvfc40vv93Evvid3vv0"d;t9r>(//5 "  4BQ7# I%34H4K3LMF'(1,O,@,C+DEEm+23G3J2KLF'(1,N+?+B*CDD 66J16M5NOF'(1,HI]^_I`Habb s##cMF RZZ(SU[\]v;~1T5O5OPV5WZ]5]r   r  c                 .   U(       d  gUR                  5       R                  5       nUR                  S5      =(       dX    UR                  S5      =(       d@    UR                  S5      =(       d(    US:H  =(       d    X R                  R                  5       :H  $ )z9Detect whether the produced answer is a generic fallback.Tr  (while i couldn't extract a direct answerz/i analyzed the available documents but couldn'tzno answer found)r   r   r  r|  )r   r  r  s      r   is_fallback_answerRAGAgent.is_fallback_answer  s    \\^))+
!!"DE @!!"LM@!!"ST@ ++@ 77==??	
r   c                 d   0 n[         R                  " 5       (       d  U$ [         R                  " S5       H  n [        US5       n[        R
                  " U5      nSSS5        [        W[        5      (       a  UnO([        U[        5      (       a  UR                  5       nOMo  U H`  n[        U[        5      (       d  M  UR                  S5      nUR                  S5      nU(       a  U(       d  ML  XaR                  U0 5      U'   Mb     M     U$ ! , (       d  f       N= f! [         a     M  f = f)zHLoad already generated knowledge cards from disk to avoid recomputation.z*_cards.jsonrNr0  card_id)r   existsglobopenjsonloadr   
isinstancer2  dictrv  rr  rt  )	r   cache
notes_pathfdatacards_iterablecardr0  r  s	            r   load_existing_card_cache!RAGAgent.load_existing_card_cache  s    68!!L#..8J*c*a99Q<D +
 $%%!%D$''!%&!$--(+((9-W8<  ,W5 ' 9. + +* s/   D!DD!
D	D!D!!
D/.D/r  thread_countc           	        ^ ^^^^^^^^ U(       d  / $ [         R                  " SSS9  ST l        T R                  5       m[	        U5      m[        ST SU S35        [        R                  " 5       m[        R                  " 5       m[        R                  " 5       m/ n0 mS/mUUUUUUUU U4	S jnUS:  a  TS:  a  [        [        USS	95      n[        US
9 nU Vs0 s H  nUR                  XX5      US   _M     n	n/ n
[        U	5       H-  nUR                  5       nU(       d  M  U
R                  U5        M/     U
R                  S S9  U
 Vs/ s H  nUS   US   S.PM     nnSSS5        O?[        USS	9 H1  u  pU" X45      nU(       d  M  UR                  US   US   S.5        M3     T R!                  T5        T R#                  T5        T R%                  T5        U$ s  snf s  snf ! , (       d  f       NM= f)a;  Create LLM-enhanced knowledge cards for each chunk and return enriched documents.

Args:
    chunks: List of chunk dictionaries to enrich
    progress_callback: Optional callback function(current_idx, total) for progress updates
    thread_count: Number of worker threads for parallel chunk processing (default: 1)
Tparentsexist_okr   u      • Starting enrichment for z chunks (using z thread(s))c                 F  >	 U u  p UR                  SS5      =(       d    Sn[        UR                  S0 5      5      nUR                  SS5      nUR                  SS5      nTR                  U5      nU(       d  UR                  5       nU SU 3nS	n	T   TR                  U0 5      R                  U5      n	S	S	S	5        U	(       a  U	n
OTR	                  UUTS
9n
UR                  U
S   U
S   U
S   U
S   U
S   S.5        TR                  X5      nUUUS.nT   UT;  a  0 TU'   U
TU   U
S   '   S	S	S	5        T   TS==   S-  ss'   TS   nT(       a
   T" UT5        US-  S:X  d  UT:X  a,  [        SU ST STR                   STR                   S3	5        S	S	S	5        U$ ! , (       d  f       N= f! , (       d  f       N= f! [         a  n[        SU 35         S	nANS	nAff = f! , (       d  f       U$ = f! [         a  n[        SU SU 35         S	nAg	S	nAff = f)z2Process a single chunk and return enriched result.rm  r   rc  r0  rd  chunk_idr   ::chunk-Nllm_usage_lockrf  rh  rg  r  clean_excerpt)rf  rh  rg  r  r  )rm  rc  r  r   z#Warning: progress callback failed: r   u      • Processed /z# chunks (knowledge-card LLM usage: )zError processing chunk r  )rr  r  r   r   generate_knowledge_cardr  compose_card_textr   r   rz   ry   )chunk_with_idxr  r  raw_textrc  r0  r  cleaned_textr  cached_cardr  enriched_contentenriched_resultcurrentr   existing_cacher  notes_by_sourcenotes_by_source_lockprocessed_countprogress_callbackprogress_lockr   total_chunkss                  r   process_single_chunk<RAGAgent.build_enriched_chunks.<locals>.process_single_chunk  sk   'JCB 99Y39r		*b 9:!h	:#<<
A6#228<##+>>#3L#HHXJ7 #)"0"4"4VR"@"D"DW"MK * &D77$ '5 8 D #I"&|"4"8n#I%)/%:!  $(#9#9$#I / (# *_424/?COF+DO< * ##A&!+&-a0G(M-g|D |q(G|,C/y, H::>:V:V9WWXY]YwYwXxxy{ # '&c *)8 *)  ) M!$Gs"KLLM #] '& /uBqc:;s   BG; "F"6A"G; F3/
G; 9G)	G;G)	G; "
F0,G; 3
G=G; 
G&G!G)!G&&G))
G83G; 8G; ;
H HH r   r  )max_workersc                     U S   $ )Nr  r  r	  s    r   r   0RAGAgent.build_enriched_chunks.<locals>.<lambda>:  s    1U8r   )r  rm  rc  )rm  rc  N)r   mkdirrz   r  r   r   	threadingLockr2  r  r   submitr   ry  r  r  save_knowledge_cardssave_document_digestssave_theme_capsules)r   r  r"  r  enriched_chunksr%  chunks_with_idxexecutor
chunk_datafuture_to_chunkresultsfuturery  r  r  r  r  r  r  r   r!  r#  r$  s   ` `             @@@@@@@r   build_enriched_chunksRAGAgent.build_enriched_chunks  s    It4'($6686{/~_\NZefg  )~~/!(")02@B#E	 E	P !q 0"9V1#=>O#= '6#&5
 OO$8EzRS}T&5   # *?;F#]]_Fvv. < !34 %#$ !")!J-H$   # >=& (a8
-sl;6#**#))#4$*:$6,  9 	!!/2""?3  1=## >=s0   G.G$<'G.')G.G)%G.$
G..
G<r  rc  c                    SUR                  SS5       SUR                  SS5       S3SUS	    3/nUS
   (       a1  UR                  S5        US
    H  nUR                  SU 35        M     US   (       a&  UR                  SSR                  US   5      -   5        US   (       a  UR                  SUS   -   5        SR                  U5      $ )zCCompose a readable knowledge card string for storage and retrieval.r  r0  rd  z	 | Chunk r  ?r
  	Summary: rf  rh  Key Points:r  rg  Themes: r  r  	Excerpt: r  rr  r  r  )r   r  rc  r  r  s        r   r  RAGAgent.compose_card_textO  s     Xy9:)HLLQ[]`DaCbbcdY()

 LL'l+r%\* , >LLdiiX&??@ LLtO'<<=yyr   r  r  c                    U R                  U5      nU(       a"  SR                  USS 5      R                  5       OUSS R                  5       nUSS  Vs/ s H  ofR                  5       PM     nnU(       d  U(       a  USS R                  5       /nU R                  U5      nUR	                  SS5       S	UR	                  S
S5       3UR	                  SS5      UR	                  S
5      UR	                  S5      USS U V	s/ s H  oSS PM	     sn	UUSS R                  5       [
        R                  " 5       R                  5       S.	n
U R                  UU
US9nU(       a  U
R                  U5        U
$ s  snf s  sn	f )z1Generate a structured knowledge card for a chunk.r   Nr   r  rt      r0  unknownr  r  r   rd  ri  r  r  r  )	r  r0  r  ri  rf  rh  rg  r  
created_atr  )
rQ  r  r   extract_candidate_themesrr  r   utcnow	isoformatenhance_card_with_modelr  )r   r  rc  r  r   rf  r  rh  rg  kpr  enhanceds               r   r   RAGAgent.generate_knowledge_cardc  sr    ..|<	5>#((9Ra=)//1LQURUDVD\D\D^/8!}=}tjjl}
=l&t,2245J..|< #,,x;<HX\\R\^_E`Dabll8Y7 Z0&ll+;<t}.89jds8j9)$3/557"//+557

 //) 0 

 KK!7 > :s   E5E:
max_themesc                 .   [         R                  " SUR                  5       5      nU(       d  / $ 1 Skn0 nU H!  nXd;   a  M
  UR                  US5      S-   XV'   M#     [	        UR                  5       S SS9nUSU  VVs/ s H  u  phUPM	     n	nnU	$ s  snnf )	z9Extract candidate themes using simple frequency analysis.z\b[a-zA-Z]{4,}\b>   rJ   rK   rL   rS   theyrT   r$  rU   rW   rZ   r[   r%  rb   rc   rg   rh   whiler   statesrl   r   becauserm   chapterhistorysectionthroughamericanro   
historicalr   r   c                     U S   $ r  r  r	  s    r   r   3RAGAgent.extract_candidate_themes.<locals>.<lambda>  s    qtr   Tr  N)r   r   r   rr  ru  r   )
r   r   rM  r   r  	frequencyr   sorted_wordsr  rg  s
             r   rF  !RAGAgent.extract_candidate_themes  s    

.

=I

 %'	D!'mmD!4q8IO 
 ioo/^TR&2;J&?@&?74$&?@ As   >B	base_cardc           	      2  ^ ^ T R                   (       a  U(       d  gS[        4UU 4S jjnU" 5       (       a  gSUSS  SUS    SS	R                  US
   5       S3n T R                  U[	        S[
        5      [        S[        5      S9nT R                  U5      nU(       a>  T(       a"  T   T =R                  S-  sl	        SSS5        U$ T =R                  S-  sl	        U$ ! , (       d  f       U$ = f! [         a  n[        SU 35         SnAgSnAff = f)z?Use the language model (Grok API) to refine the knowledge card.Nr   c                     > T (       a%  T    TR                   TR                  :  sS S S 5        $ TR                   TR                  :  $ ! , (       d  f       N'= fr$  )rz   ry   )r  r   s   r   budget_exceeded9RAGAgent.enhance_card_with_model.<locals>.budget_exceeded  sC    #774;Y;YY $^//43Q3QQQ $^s   A
AzjYou are refining a knowledge card for a document chunk. Extract the most important information.

EXCERPT:
r]  z

CURRENT SUMMARY:
rf  z

CURRENT KEY POINTS:
; rh  a  

Provide an improved knowledge card in the following exact format:
Summary: <rewrite into 1-2 sentences capturing the main point>
Key Points:
- <concise bullet 1>
- <concise bullet 2>
- <concise bullet 3>
Themes: theme1, theme2, theme3

Begin the card now:
r  r   r  r   z,Warning: knowledge card enhancement failed: )r{   boolr  r   r   r
   rx   r   parse_knowledge_card_outputrz   r   r   )	r   r  r^  r  ra  r   	card_textparsedr   s	   `  `     r   rI   RAGAgent.enhance_card_with_model  s4    ..l	R 	R 	R  ds   9   9\"# $ %,	66sN3[1 7 I 55i@F!'4494 ( M 00A50M	 ( M 	@DE	s=   AC4 +C"	C4 C4 "
C1,C4 1C4 4
D>DDc                    U(       d  g[         R                  " SU5      n[         R                  " SU[         R                  5      n[         R                  " SU5      nU(       d  gUR                  S5      R	                  5       n/ nU(       a  UR                  S5      R                  5       nU H|  nUR	                  5       nU(       d  M  UR                  5       R                  S5      (       a    O>UR                  S5      (       d  MZ  UR                  USS R	                  5       5        M~     U(       d
  U(       a  U/n/ n	U(       aV  UR                  S5      n
U
R                  S5       Vs/ s H)  oR	                  5       (       d  M  UR	                  5       PM+     n	nUSS	 USS
  Vs/ s H  oSS PM	     snU	SS S.$ s  snf s  snf )z7Parse LLM output into structured knowledge card fields.NzSummary:\s*(.+)zKey Points:\s*(.*)zThemes:\s*(.+)r   zthemes:r   ,r  rt   r  rV  )rf  rh  rg  )
r   searchr   r   r   
splitlinesr   r  r  r   )r   r   summary_matchkey_points_sectionthemes_matchrf  rh  r  r  rg  themes_textr  rJ  s                r   re  $RAGAgent.parse_knowledge_card_output  s{   		"4d;YY'<dBIINyy!2D9%%a(..0 "
&,,Q/::<Ezz|::<**955??3''%%d12hnn&67  g!J&,,Q/K1<1B1B31GY1G;;=mekkm1GFY t}.8!n=nds8n=Raj
 	
 Z >s   ,G G )Gr  c           	      `   UR                  5        GH`  u  p#[        U5      R                  =(       d    Sn[        U S3-  n0 nUR	                  5       (       a   [        US5       n[        R                  " U5      n[        U[        5      (       a:  U V	s0 s H,  n	[        U	[        5      (       d  M  U	R                  S5      U	_M.     nn	O[        U[        5      (       a  UnSSS5        UR                  U5        UR                  R                  SSS9  [        US5       n[        R                   " [        UR#                  5       5      US	S
9  SSS5        [%        S['        U5       SU 35        GMc     gs  sn	f ! , (       d  f       N= f! [         a    0 n Nf = f! , (       d  f       NV= f)z5Persist knowledge cards to disk for future reference.unknown_sourcez_cards.jsonr  r  NTr
  r  r   indentu      • Saved z knowledge cards for )r   r   r  r   r  r  r  r  r  r2  r  rr  r   r  parentr*  dumprv  r   r   )
r   r  r0  cardssource_stemr  existing_cardsr  r  r  s
             r   r.  RAGAgent.save_knowledge_cards  si   ,224MFv,++?/?K"}K%@@J8:N  ""(j#.!#yy|%dD11TX-sTXD\fgkmq\r.Gdhhy.A4.GTXN-sN'd33-1N / !!%(##D4#@j#&!		$~44671E 'M#n"5!66KK=YZ+ 5 .t /. ! (%'N( '&sT   F%0E<E72E7E<"F!.F7E<<
F
	F
FFF
F-	c                    U(       d  g[         R                  " SSS9  UR                  5        H  u  p#U R                  U[	        UR                  5       5      5      nU(       d  M7  [        U5      R                  =(       d    Sn[         U S3-  n [        US5       n[        R                  " XGSS9  SSS5        X@R                  U'   M     g! , (       d  f       N!= f! [         a  n[        S	U S
U 35         SnAM  SnAff = f)z+Create and save distilled document digests.NTr
  rs  z_digest.jsonr  r   rt  z#Warning: failed to save digest for r  )r   r*  r   build_document_digestr2  rv  r   r  r  r  rw  r~   r   r   )	r   r  r0  rx  rz  ry  digest_pathr  r   s	            r   r/  RAGAgent.save_document_digests!  s    $6,224MF//U\\^8LMFv,++?/?K%;-|(DDKP+s+qIIf2 ,06%%f- 5 ,+  P;K=1#NOOPs0   CC2C
C	C
D(C??Dr0  rx  c           
        ^% U(       d  g/ n0 n/ SQnU Hl  nUR                  S5      =(       d    SR                  5       nU(       d  M4  UR                  5       SS R                  5       nUR                  US5      S-   XH'   Mn     U GHH  nUR                  S5      =(       d    SR                  5       nU(       a  U R                  U5      OSn	Sn
UR                  5       m%U H  nUT%;   d  M  U
S	-  n
M     UR                  5       SS R                  5       nSnX;   a  XL   nUS:  a  [	        S
US-  5      nUR                  SS5      n[	        S	US-  5      n/ SQnSn[        U%4S jU 5       5      n[	        SUS-  5      nUR                  S/ 5      nU(       a  [        U5      S:  a  US-  nU	U-   U-
  U
-
  U-
  nUR                  UU45        GMK     UR                  S SS9  U VVs/ s H  u  nnUS:  d  M  UPM     snnSS nU(       d  USS nSn/ n/ n0 n/ nU GH@  nUR                  S5      =(       d    SR                  5       nU(       a  UR                  U5        UR                  S/ 5      SS  H+  nU(       d  M  UR                  UR                  5       5        M-     UR                  S/ 5       H+  nUR                  5       nUR                  US5      S-   UU'   M-     UR                  S5      nU(       a  UR                  U5        U(       a  M  UR                  S5      =(       d!    UR                  S0 5      R                  S5      n U (       d  GM>  U nGMC     U R                  U5      SS nU R                  U5      SS n[        UR                  5       S SS9SS  VV!s/ s H  u  nn!UPM
     n"nn!S R                  U5      n#U#(       d  U(       a  S R                  USS 5      n#U#(       d  gUU=(       d%    [        U5      R                  R                  S!S 5      U#SS" UUU"USS [        U5      [         R"                  " 5       R%                  5       S#.	n$U$$ s  snnf s  sn!nf )$z@Create a concise digest for a document from its knowledge cards.N)zthe secret doctrineztheosophical university presszonline editionzsynthesis of sciencereligionzand philosophyzthis edition was correctedzfacsimile editionzobvious errorszmissing letterszauthor feelszapologise for the long delayzindulgence of her readerszzebra on nextzsan francisco public libraryzdue datesfplzcornell university libraryzsage endowment fundzthe gift ofrf  r   r  r   r   r   r   rR  r   r  g{Gz?)&rE  r2  r6  dragonr4  initiateadept	mythologyr  r   atlantisr   r'  r1  r  r  supportr:  	financiercelticogaminscriptionr   r  r  r   r  r  r   r   r  lawcrownr  r  	principlemoralconstitutionc              3   6   >#    U  H  oT;   d  M
  S v   M     g7fr   Nr  )r   keywordsummary_lowers     r   r   1RAGAgent.build_document_digest.<locals>.<genexpr>n  s     !`3CR_G_!!3Cs   		rS  g333333?rh  c                     U S   $ r  r  r	  s    r   r   0RAGAgent.build_document_digest.<locals>.<lambda>z  s    !r   Tr  r  rN  r   rg  r  ri  rc  rV  r  c                     U S   $ r  r  items    r   r   r    s    PTUVPWr   r   r  i  )	r0  rl  rk  snippetsrh  rg  card_ids
card_count
updated_at)rr  r   r   r   r   r   r   r  r  r  ru  r   r  r   r  r   r   rG  rH  )&r   r0  rx  scored_cardssummary_countsboilerplate_phrasesr  rf  r  r   boilerplate_penaltyr%  normalized_summaryrepetition_penaltyr   r  position_penaltycontent_keywordsboost_scorekeyword_matchesrh  total_scorer  	top_cardsri  overview_sentencestheme_countsevidence_idsr  r  
norm_themer  
meta_titler  
top_themesrk  rz  r  s&                                        @r   r}  RAGAgent.build_document_digest6  sf     
 Dxx	*0b779Gw$]]_Tc288:
-;-?-?
A-NQR-R*  Dxx	*0b779GCJD66w?PSM #&#MMOM-]*'3.' .
 ")#!6!<!<!>!$!3&:19),S%#+)>& xx
A.H"348 ^ K!!`3C!``Oc?T#9:K ,3Jc*o1s"'+58JJM``cssKd 34Q V 	nd;-9I\keTUS[T\I#2N	cr
I(* "
')"$Dxx	*0b779G"))'2,3BQ75%%ekkm4 8 (B/"[[]
+7+;+;J+JQ+NZ( 0 hhy)G##G,!>!XX&67i488JPR;S;W;WXh;i
:%/N) , "223EFrJ**:6r:
"(););)=CWae"fgihi"j
"jheQE"j 	 
 88./Jxx
2A/H #JtF|'8'8'@'@c'J #*$ $Sb)e*"//+557

 w JH
s   )Q#:Q#,Q)c                    U(       d  g[         R                  " SSS9  0 nUR                  5        GH"  u  p4UR                  5        GH  nUR	                  S5      =(       d    / nU(       d  M'  UR	                  S5      =(       d    SR                  5       nUR	                  S5      =(       d    / nU H  n	U	R                  5       n
UR                  U
U	/ / [        5       S.5      nU(       a  US	   R                  U5        USS
  H.  nU(       d  M  US   R                  UR                  5       5        M0     US   R                  U5        M     GM
     GM%     U(       d  gU R                  5       nUR                  5        H  u  pUR	                  U
US   / / / SS.5      nU R                  UR	                  S	/ 5      US	   -   5      SS US	'   U R                  UR	                  S/ 5      US   -   5      SS US'   [        UR	                  S/ 5      5      US   -  n[        U5      SS US'   [        R                  " 5       R!                  5       US'   XU
'   M     [         S-  n [#        US5       n[$        R&                  " UUS
S9  SSS5        Xl        g! , (       d  f       N= f! [*         a  n[-        SU 35         SnAgSnAff = f)z>Aggregate theme capsules across documents for quick reference.NTr
  rg  rf  r   rh  )r  r  rh  r  r  r   r  r  )r  r  rh  r  r  rV  r  rQ  r  capsules.jsonr  rt  z(Warning: failed to save theme capsules: )r   r*  r   rv  rr  r   r   rt  r  r  r  r   r  ru  r   rG  rH  r  r  rw  r   r   r   )r   r  theme_updatesr0  rx  r  rg  rf  rh  r  	theme_keyentryr  capsulesr  merged_sourcescapsules_pathr  r   s                      r   r0  RAGAgent.save_theme_capsules  s   4$735,224MF(+1r88I.4";;=!XXl39r
#E %I)44Y!&$&&(#&5	A E j)009!+BQ 5!,/66u{{}E "0 )$((0 $ ' 50 ++-!.!4!4!6ILL "- E !% 5 5eii
B6ORXYcRd6d efhgh iE*"&"7"7		,PR8SV\]iVj8j"klnmn"oE, 9b!9:VI=NNN%n5cr:E)"*//"3"="="?E,"'Y "7" %6	BmS)Q		(Aa0 *"* *)  	B<QC@AA	Bs0   #J& /JJ& 
J#J& &
K0KKc                    0 n[         R                  " 5       (       d  U$ [         R                  " S5       Hd  n [        US5       n[        R
                  " U5      nSSS5        [        W[        5      (       d  MF  UR                  S5      nU(       d  M`  XAU'   Mf     U$ ! , (       d  f       NI= f! [         a     M  f = f)z'Load stored document digests from disk.z*_digest.jsonr  Nr0  )
r   r  r  r  r  r  r   r  r  rr  )r   digestsr~  r  r  r0  s         r   r   RAGAgent.load_document_digests  s    -/!!##N&++O<K+s+q99Q<D ,
 $%%(+6&*FO =  ,+ s/   B0BB0
B-	)B0-B00
B>=B>c                 2   [         S-  nUR                  5       (       d  0 $  [        US5       n[        R                  " U5      nSSS5        [        W[        5      (       a  U$ [        U[        5      (       aa  U Vs0 s HS  n[        U[        5      (       d  M  UR                  S5      (       d  M2  UR                  SS5      R                  5       U_MU     sn$  0 $ ! , (       d  f       N= fs  snf ! [         a  n[        SU 35         SnA0 $ SnAff = f)zLoad theme capsules from disk.r  r  Nr  r   z(Warning: failed to read theme capsules: )r   r  r  r  r  r  r  r2  rr  r   r   r   )r   r  r  r  r  r   s         r   r   RAGAgent.load_theme_capsules  s    $6##%%I	BmS)Qyy| *$%%D$'' "&!%!%. :3899W3E :EIIgr*002E9!%  ( 	 *)

  	B<QC@AA		BsL   C3 CC3 $C3 =C.C.2%C.C3 
C+'C3 3
D=DDr   c                     [        5       n/ nU H^  nU(       d  M  UR                  5       nU(       d  M%  UR                  5       nXb;   a  M<  UR                  U5        UR	                  U5        M`     U$ )z<Remove duplicates while preserving order (case-insensitive).)r  r   r   r  r  )r   r   r  ry  r  r  r  s          r   r  RAGAgent._dedupe_preserve   sc    uDJ""$C{HHSMMM*%  r   c           	         0 n/ nU R                   R                  5        H  u  p4/ nUR                  S5      nU(       a  UR                  U5        UR	                  UR                  S5      =(       d    / 5        UR	                  UR                  S5      =(       d    / 5        U H  nU(       d  M  UR                  Xs45        M      M     U R
                  R                  5        H|  nUR                  S5      =(       d    Sn	SU	 3n
UR                  S5      =(       d    / nUR                  S5      =(       d    / nX-    H  nU(       d  M  UR                  Xz45        M      M~      U R                  5       nUR                  5        H  u  p>UR                  5        H  nUR                  S5      nU(       a  UR                  UU45        UR                  S5      nU(       a  UR                  UU45        UR                  S5      =(       d    /  H  nU(       d  M  UR                  UU45        M!     M     M     U H  u  psU R                  U5      n[        U5      S:  a  M'  U R                  U5      nU R                  U5      S
S nU Hk  nUR                  U/ / [        5       S.5      nU(       a/  UUS   ;  a&  [        US   5      S:  a  US   R                  U5        US   R!                  U5        Mm     [#        US5       H0  u  nnU R%                  UUUUU5        U R%                  UUUUU5        M2     GM      UR                  5        GH  u  nnU R                  UR                  S/ 5      5      S
S US'   / n[        5       nUR                  S/ 5       H  nUR                  S5      nUR                  SS5      n UU 4n!U!U;   d  U(       d  M9  UR!                  U!5        U R                  U 5      S
S US'   UR                  U5        [        U5      S:  d  M    O   UUS'   ['        [        UR                  S/ 5      5      5      S
S US'   GM     U$ ! [         a  n[        S	U 35         S
nAGN;S
nAff = f)z>Construct a lightweight entity graph from distilled resources.rk  rh  r  r  zTheme CapsulezTheme Capsule: rf  r  z:Warning: unable to load knowledge cards for entity graph: Nr   r  r  connectionsr  r  rV  r  r  entityr^  r   i  r   )r~   r   rr  r  r[  r   rv  r  r   r   _extract_entities_from_textr   r  r   rt  r  r  r   _add_entity_connectionru  )"r   graphsegmentsr0  rz  entriesrk  r   capsuletheme_labelcapsule_sourcer  rh  
card_cacherx  r  rf  excerptr  excr  unique_entitiescleaned_snippetr  nodeleftr   r  r  
seen_edges
connectiontargetr^  r  s"                                     r   r   !RAGAgent.build_entity_graph_index1  s   +-*, #3399;NFGzz*-Hx(NN6::l39r:NN6::j17R84OOTN3   < **113G!++g.A/K.{m<N{{:.4"H \28bJ -4OOT$:; . 4	V668J!+!1!1!3!LLND"hhy1G &(9:"hh7G &(9:!%,!7!=2!= 5$OOUFO< "> + "4 %LD77=H8}q "33H=O"11$7=O)''!##%"u1 
 #d;>O'OTWX\]hXiTjmnTn%,,_=Y##F+ *  ,OQ?e++E4QWX++E5$QWX  @% %, "KKMLFD $ 5 5dhh{B6O PQSRS TDKJ"hh}b9
#1$..B7w'*$Fs#(,(;(;G(DTc(J
9%"":.{#q( : #.D$S)R)@%AB3BGDO! *$ W  	VNseTUU	Vs   $B0Q Q 
Q7Q22Q7r  r  r  r^  c                     U(       a  U(       a  X#:X  a  g UR                  U/ / [        5       S.5      nUS   R                  UU=(       d    SUS.5        g )Nr  r  r   )r  r^  r0  )rt  r  r  )r   r  r  r  r^  r0  r  s          r   r  RAGAgent._add_entity_connection  sZ     Vv'7u)
 
 	]""}"$
 	r   c                   ^
 U(       d  / $ 1 SknU Vs1 s H  o3R                  5       iM     snm
[        R                  " SU5      nU[        R                  " SU5      -  n/ nU H  nUR                  5       n[	        U5      S:  a  M$  Xb;   a  M+  UR                  5       nUT
;   a  MC  UR                  5       n[        U
4S jU 5       5      (       a  Mo  UR                  5       U:X  a  [	        U5      S::  a  M  UR                  SS5      R                  5       n	U	(       d  M  UR                  U	5        M     U R                  U5      $ s  snf )z0Extract capitalised entity candidates from text.>   ThePageThatThisTheseThoseVolumeChapterEditionLibrarySection
Conclusion
UniversityIntroductionz&\b(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\bz\b[A-Z]{3,}\br   c              3   H   >#    U  H  oR                  5       T;   v   M     g 7fr$  r  )r   tokenstopwords_lowers     r   r   7RAGAgent._extract_entities_from_text.<locals>.<genexpr>  s     H;;=O3s   "r   r   )r   r   r   r   r   r   r1   rv  r   r  r  )r   r   	stopwordsr   
candidatesr  	candidatecandidate_lowertokensrO  r  s             @r   r  $RAGAgent._extract_entities_from_text  s   I
	 5>>ID::<I>ZZ I4P
bjj!1488
 #I!)I9~!%'oo/O/1__&FHHHH I-#i.A2E''c288:GOOG$) $, $$X..9 ?s   Emax_entriesc                 4   U(       a  U R                   (       d  gUR                  5       n[        [        R                  " SU5      5      n/ nU R                   R                  5        GH!  u  pgUR                  5       n[        [        R                  " SU5      5      n	UR                  U	5      n
[        SX85      R                  5       nU[        U
5      S-  -   nX;   a  US-  nUR                  S/ 5       Hs  nUR                  SS5      R                  5       nU(       d  M,  X;   a  US-  n[        [        R                  " SU5      5      nUR                  U5      (       d  Mn  US	-  nMu     US
:  a  GM  UR                  XU45        GM$     UR                  S SS9  / nUSU  H  u  npgSnUR                  S5      =(       d    / nU(       ag  US   nUR                  SS5      nU R                  UR                  SS5      5      SS nUR                  S5      nU SU SU 3nU(       a	  USU S3-  nO:UR                  S5      (       a"  U R                  US   S   5      SS nU SU 3nOUnUR                  SU 35        M     SR                  USU 5      $ )zKProduce a short list of entity-relationship hints relevant to the question.r   r  Ng333333?g      ?r  r  rR  r  r  c                     U S   $ r  r  r  s    r   r   5RAGAgent._format_entity_graph_hints.<locals>.<lambda>  s    T!Wr   Tr  r   Relatedr^  r  r0  u    ↔ r  z
 (Source: r  r  r  r  )r   r   r  r   r   r   r  r   rI  r   rr  r  r  r   r  )r   r!  r  r&  r  r  r  r  entity_lowerentity_wordsr  rk  r  r  target_lowertarget_wordshintsr  connection_textr  top_connrelatedr^  r0  s                           r   r  #RAGAgent._format_entity_graph_hints  sv   t00!)RZZ
NCD:< --335LF!<<>Lrzz*lCDL$11,?G(~LRRTJW!33E-"hh}b9
)~~h;AAC#1SLE"2::j,#GH!..|<<SLE : s{MM5$/0+ 6. 	,d;%l{3OAv O((=17RK&q>",,x;--hll9b.IJ4CP!h/%+HE'"WI"F#F81'==O+&&--d;.?.BCDSI%+HBwi"8"(LL2o./0!  4$ yy|,--r   c                 >   UR                  S5      n/ nU H  nUR                  5       (       d  M  UR                  S5       Vs/ s HK  oUR                  5       (       d  M  UR                  S5      (       a  M1  UR                  S5      (       a  MI  UPMM     nnSR                  U5      n[	        UR                  5       5      S:  a  M  [        S U 5       5      n[	        U5      n	U	S:  a
  X-  S:  a  M  UR                  U5        M     SR                  U5      $ s  snf )	z+Filter context to remove low-quality chunksr  r  httpr  r   c              3   R   #    U  H  oR                  5       (       d  M  S v   M     g7fr  )isalnumr   cs     r   r   2RAGAgent.filter_context_quality.<locals>.<genexpr>	  s     @AIIKaar  r   rS  )r   r   r  r  r   r   r  )
r   r   r  r  r  r  r  rm  alpha_counttotal_counts
             r   r  r     s    y)E;;== /4kk$.?  N.?d::<TX\XgXghnXoTx|  yH  yH  IL  yMT.?M  Nii.G 7==?#b( @@@Kg,KQK$=#D""5)% ( ~~o.. Ns   DD7DDrm  c                    SnSUR                  5       ;   d  SUR                  5       ;   a  US-  nU(       a  [        S U 5       5      [        U5      -  OSnUS:  a  US-  n[        R                  " S	U5      n[        U5      S
:  a  US-  n[        UR	                  5       5      nSUs=::  a  S::  a	  O  OUS-  nOUS:  a  US-  n[        S[        S
U5      5      $ )z!Score content quality (0-2 scale)r   r  z.pdfr   c              3   4   #    U  H  oS ;   d  M
  Sv   M     g7f)z.,!?;:r   Nr  r  s     r   r   r   	  s     D7a8m7s   		r   r   r   r   r   rN  rC  r   r  )r   r   r   r   r   rx   r   )r   rm  r  punctuation_ratior   r   s         r   r   r   	  s     W]]_$'--/(ASLE X_CD7DDs7|Sdes"SLE HHY0	y>QSLE )
"s"SLE"_SLE1c!Um$$r   c                    U R                  U5      nU(       a  UR                  5       n/ n/ n/ SQnU H  n	U	R                  S0 5      R                  SS5      R                  5       n
U	R                  SS5      R                  5       nSnU H  nX;   d  M
  X;   d  X;   d  M  Sn  O   U(       a  UR                  U	5        M  UR                  U	5        M     Xg-   nU R	                  U5      nU R                  X.U5      nU(       a  U$ US:X  a  U R                  X!U5      nU(       a  U$ U R                  U5      nUU R                  :g  nU R                  X!U5      nU(       a5  [        UR                  5       5      S	:  a  U R                  U5      (       d  U$ U R                  UUUU(       a  UOS
US9nU(       a  UU R                  :w  a  U$ U(       a  U$ U R                  $ )zHMain agentic method - prioritize direct extraction over model generation)	r  r  r  r  r  zchristopher columbusr   r  r  rc  r0  r   rm  FTr)  r  N)r  )rD  r   rr  r  r  r  r   r}  r|  r  r   r   r  r  )r   r   r!  rw  r  r&  prioritized_resultsother_resultsspecific_namesry  r0  rm  relevantnamer  digest_answersource_answerfallback_candidater  r  r  s                        r   generate_extraction_answer#RAGAgent.generate_extraction_answer/	  s    --h7 %^^-N"$MQN )J377"EKKM **Y399; !*D-4>T_#' +
 '..v6!((0 )" 1@N!<<^L;;HZgh  O+ <<XP^_M$$ "55h? 2d6S6S S 11(]SS!4!4!67"<TE\E\]jEkEk   ))"7T1 * 
 f = ==M !%%,,,r   c                 z   [         R                  " SSU5      n[         R                  " SSU5      n[         R                  " SSU[         R                  S9n[         R                  " SSU[         R                  S9n[        U5      S:  Ga[  UR	                  5       n[        U5      S	:  Ga;  S
UR                  5       ;   a  UR                  5       R                  S
5      S:  a}  UR	                  S
S5      n[        U5      S:  a[  US   R                  5       n[         R                  " SU5      nUS   R                  5       US   R                  S5      (       d  SOS-   nO[        [        USS	 5      5      S:  al  [         R                  " SU5      nU Vs/ s H2  n[        UR                  5       5      S:  d  M"  UR                  5       PM4     snSS nU(       a	  US   S-   nOSn[        UR                  5       5      S:  d  [        S U 5       5      (       d  gSSSSS.nUR                  US5      n	[        U5      U	:  ae  USU	 R                  5       nUR                  S5      (       d<  UR                  S5      n
XS-  :  a  USU
 S-   nUR                  5       $ USU	S-
   S-   nUR                  5       $ s  snf )z)Clean up and improve the generated answerr   r   r   z
Answer:\s*r   r   zStep-by-step reasoning:\s*r   r   zanswer:r   r   r   r   ru  Nr   rQ  zqThe Nagas were semi-divine beings in Hindu mythology, often depicted as serpents or having serpent-like features.c              3   @   #    U  H  oR                  5       v   M     g 7fr$  )r   r  s     r   r   /RAGAgent.post_process_answer.<locals>.<genexpr>	  s     .KFqyy{{F   zAccording to the documents, the Nagas were ancient serpent-like beings or deities in Hindu and Buddhist mythology, often associated with wisdom, water, and sometimes depicted as having human upper bodies with serpent lower bodies.i,    r  i^  r4  r;  r@  rA  rR  rV  )r   r   r   r   r   r   r   r   r  r  r3   rr  rfind)r   r  r  r   partsr   r   rx  max_lengthsrT  
last_spaces              r   r  RAGAgent.post_process_answer|	  ss    V,V, r6G5r6W v;LLNE5zB.6<<>3G3G	3RUV3V"LLA6E5zA~!&q!1$&HHY$?	!*1!3!3!5IaLDYDYZ]D^D^df!g U3BZ)A- "F ;I?H+`y!CPQPWPWPYN]_L_IAGGIy+`acbc+d(+!5a!83!> "U v||~#3.KF.K+K+K | 	
 !__]C8
v;#KZ(..0F??3''#\\#.
S 00#KZ058F ||~ $MZ\2U:F||~9 ,as   !J8:J8c                 Z    U R                  XU5      nU(       a  U R                  U5      nU$ )z?Main method to generate answer using extraction-based reasoning)r  r  )r   r   r!  rw  r  s        r   generate_answerRAGAgent.generate_answer	  s,    00NS..v6Fr   c                 
  ^ U(       d  gUR                  5       mSSKJn  [        U[        SS9n[        U4S jU 5       5      n/ SQn[        U4S jU 5       5      n/ S	Qn[        U4S
 jU 5       5      n/ SQn	[        U4S jU	 5       5      n
/ SQn[        U4S jU 5       5      n/ SQn[        U4S jU 5       5      n[        UUUU
U/5      nU=(       d)    U=(       d     U=(       a    U=(       d    U
=(       d    US:  nU$ )z?Enhanced detection of technical queries using multiple signals.Fr   TECHNICAL_TERMSTr  c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r,  s     r   r   3RAGAgent._detect_technical_query.<locals>.<genexpr>	  s      QLD!7Lr(  )u,   \d+\.?\d*\s*(ph|ppm|ec|tds|°[cf]|percent|%)z3(ph|temperature|humidity|ppm|ec|tds)\s*[:\-]?\s*\d+u3   \d+\s*[-–—to]\s*\d+\s*(ph|ppm|°[cf]|percent|%)c              3   R   >#    U  H  n[         R                  " UT5      v   M     g 7fr$  r   rk  r   patternr&  s     r   r   r,  	  s!     ePdWbii@@Pd   $')z6what (is|are) (the|an?|optimal|ideal|recommended|best)zhow (much|many|often|long)z what (should|must|can) (the|an?)zP(optimal|ideal|recommended|best|minimum|maximum) (ph|temperature|humidity|light)c              3   R   >#    U  H  n[         R                  " UT5      v   M     g 7fr$  r.  r/  s     r   r   r,  	  s!     #rVq7BIIg~$F$FVqr1  )measuretestcheckmonitoradjust	calibratemaintainc              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  )r   verbr&  s     r   r   r,  	  s      TOD!7Or(  )	metertestergaugesensorlightledhpsmhballastc              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r,  s     r   r   r,  	  s     OtN2r(  )&phzph levelzph.zph-zph:zph scaleppmectdsr   humiditynutrient
fertilizernpkdosagedosemgcbdthccannabinoidtherapeuticzclinical trialzside effectlegalillegal
compliancelicensezpossession limitrevenueprofitroizmarket size
investmentstudyresearchzpeer-reviewedstatisticalzp-valuec              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r,  s     r   r   r,  	  s     XAWn4AWr(  r   )r   configr*  ru  r   r3   r   )r   r!  r*  sorted_termshas_technical_termmeasurement_patternshas_measurementtechnical_question_patternshas_technical_patterntechnical_verbshas_technical_verbequipment_termshas_equipmentstrong_technical_termshas_strong_termsignal_countis_technicalr&  s                    @r   rB   RAGAgent._detect_technical_query	  s   !) 	+ o3E  QL QQ 

 ePdee'
# !$#rVq#r r e  TO TT iOOO"
 XAWXX !
   Q%:%P>P A	 	 r   c           	      .  ^,^-^. U(       d  g[        US SS9nU(       a  U R                  U5      OSnSSKJn  Um.[	        5       nU(       a8  UR                  5       n[	        [        R                  " SU5      5      n1 S	kn	Xy-
  n/ n
Sn[	        5       nSnU GH  nUR                  S
0 5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  5       m,UR                  S/ 5      nU(       a  U=(       a    [        U.4S jU 5       5      nOU=(       a    [        U,4S jT. 5       5      nU=(       a    [        U,4S jU 5       5      nUR                  SS5      =(       d)    U=(       a     [        [        R                  " SU5      5      nUR                  SS5      nU(       d  [        U,4S jS 5       5      nU(       a  U(       a
  U(       d  GMX  U R                  R                  U5      nU(       a  U R                  U5      nGOpU(       a  U(       d  U(       a  U(       d  U R                  U5      nU R                  U5      n/ nU H}  nUR                  5       m-U(       a-  [        U-4S jU 5       5      (       a  UR!                  U5        MG  U(       d  MP  [        U-4S jT. 5       5      (       d  Ml  UR!                  U5        M     U(       a  SR#                  USS 5      nO U(       a  SR#                  USS 5      OUSS nSnGOmUR                  S 5      =(       d    SnUR                  S!5      =(       d    / nUR                  S"5      =(       d    / nUR                  S#5      =(       d    Sn U(       dC  U R                  U5      n!U R                  U!5      nU(       a  SR#                  USS$ 5      nOU!SS% nS&UR%                  5        3/n"U(       a1  U"R!                  S'5        USS(  H  n#U"R!                  S)U# 35        M     U(       a&  U"R!                  S*S+R#                  USS 5      -   5        U (       a*  U R                  U 5      n$U$SS, n$U"R!                  S-U$ 35        S.R#                  U"5      nU(       d  GM  UR                  S/5      =(       d"    U=(       a    U=(       d    U=(       a    U(       + n%U%(       aA  U R'                  UUS0S19n&U&(       a)  S.R#                  U&SS$ 5      n'UR%                  5        S2U' 3nU(       d  GM  S3U S4US5 S6UR%                  5        S.3n([)        U(5      n)UU)-   U:  a  GM  U
R!                  U(5        UU)-  nUR+                  U5        GM     U(       aa  U R-                  X,5      n*U* HJ  n+S7U+R%                  5        S.3n(U[)        U(5      -   U:  a    O#U
R!                  U(5        U[)        U(5      -  nML     U
(       d  U R/                  XC5      $ S8R#                  U
5      $ )9z;Format search results into distilled context for the model.r   c                     U S   $ ro  r  r	  s    r   r   )RAGAgent.format_context.<locals>.<lambda>
  rq  r   Tr  Fr   r)  r  >   r    r!   r#   r%   r(   r)   r,   r-   r0   r4   r8   r;   r@   rA   rC   rU   rV   rW   rZ   rg   rc  r0  rd  r  r   rm  technical_termsc              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  )r   r-  rt  s     r   r   *RAGAgent.format_context.<locals>.<genexpr>7
  s     9t^sVZ/:Q^sr(  c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r   r-  r  s     r   r   rv  9
  s     9l\kTX-:O\kr(  c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  rx  s     r   r   rv  ;
  s     1`T_D-2GT_r(  has_measurementsz	\d+\.?\d*is_intro_chunkc              3   2   >#    U  H  oTS S ;   v   M     g 7f)Nr  r  )r   
intro_termr  s     r   r   rv  D
  s%      % Z:=#3F%F Zs   )zthis book is writtenzputting aside any legalzintroduction cannabisprefaceforeword	copyrightzpublished by
disclaimerztable of contents
dedicationacknowledgmentc              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r   r-  rd  s     r   r   rv  [
  s     .^R]$~/ER]r(  c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  r  s     r   r   rv  ]
  s     1eUdT.2HUdr(  r   Nr  rV  r]  rf  rh  rg  r  r   r  r<  r=  rt   r  r>  r  r  r?  r  has_keyword_matchr   )windowz

Adjacent Context:
r  , Relevance: .3f]
z[Theme Capsule]
r  )ru  rB  ra  r*  r  r   r   r   rr  r3   rd  rk  r~   format_digest_for_contextr   rQ  r  r  r   _get_neighbor_contextr   r  find_relevant_theme_snippetsbuild_raw_context)/r   rw  r!  rT  sorted_resultsro  r*  query_termsquery_lowerr  context_partscurrent_lengthused_sourcestechnical_chunks_foundry  rc  r0  r  raw_contentchunk_technical_termshas_technical_infohas_query_termsrz  r{  rz  rm  r   relevant_sentencesrP  rf  rh  rg  r  rO  r  r  excerpt_cleanshould_expand_contextneighbor_snippetsneighbor_text
chunk_textchunk_lengththeme_snippetsr^  r  rd  rt  s/                                               @@@r   format_contextRAGAgent.format_context
  s   4HRVW BJt33H=u 	+) e"..*Kbjj[ABK bJ%2K#%u!&$Fzz*b1H\\(I6FJJw,E **Y3K (--/M$,LL1BB$G!$%1%tc9t^s9t6t"%1%lc9l\k9l6l")`c1`T_1`.`O  (||,>F   H<  LG\`acajajkw  zE  bF  ]G &\\*:EBN!!$ % Z % " "8AS**..v6F88@ !%7_Uc"11+>G $ : :7 CI)+&$-)1)9&3.^R].^+^+^.55h?)\c1eUd1e.e.e.55h? %. *"%((+=bq+A"B >G#((9Ra="9GTXUXM-1* 'll95;G!)l!;!ArJ%\\(39rF&ll?;ArG""&"5"5k"B$($>$>w$G	$&)hhy!}&=G&-dsmG $GMMO#45E "]3%/^E!LL2eW6 &4 Z$))F2AJ2G%GH(,(;(;G(D(5ds(;y%@A"ii.G 

./ 94"49 7%7 "
 %$($>$>vxXY$>$Z!$$(II.?.C$DM!( 11HXG$VHM%CGXXZ[Jz?L,z9  ,l*NV$m %p !>>xVN)00AD
!C
O3j@$$Z0#j/1 * )).EE~~m,,r   c                 p   U(       d  gU R                   nX;   a  X!   $ [        U5      R                  S-   n[        [        5      U-  nUR	                  5       (       d  SX!'   g [        US5       n[        R                  " U5      nSSS5        0 n[        W5       HA  u  pU
R                  S5      =(       d    0 nUR                  SU	5      n [        U5      nXU'   MC     SU0X!'   X!   $ ! , (       d  f       Nj= f! [         a  n[        SU SU 35        SX!'    SnAgSnAff = f! [         a     M  f = f)	z6Load chunk metadata for a given source PDF into cache.Nz_chunks.jsonr  z#Warning: failed to load chunk file r  rc  r  id_map)r   r   r  r   r  r  r  r  r   r   r  rr  int)r   r0  r   chunk_filename
chunk_pathr  r  r   r  r  r  rc  r  s                r   _load_chunk_cache_entry RAGAgent._load_chunk_cache_entry
  s3   !!?= f**^;-(>9
  "" EM	j#&!1 ' -/#F+JCyy,2H||J4Hx=  %8 , "6*}% '& 	7
|2aSIJ EM	  sH   #C; /C*C; D'*
C84C; 8C; ;
D$DD$'
D54D5r  c                    U(       d  / $ UR                  S5      nUc  / $  [        U5      nU R                  U5      nU(       d  / $ UR                  S5      =(       d    0 n/ n[	        U* US-   5       H  nUS:X  a  M  XH-   n	UR                  U	5      n
U
(       d  M)  U
R                  S5      =(       d!    U
R                  S0 5      R                  S5      nU(       d  Mk  U R                  U5      nU(       d  M  U
R                  S0 5      R                  S	5      nS
U	 3nUb  USU 3-  nUS-  nUR                  XSS -   5        M     U$ ! [         a    / s $ f = f)z=Return nearby chunk excerpts around the given chunk metadata.r  Nr  r   r   rm  rc  r  pagez(Chunk z, Page z) r  )rr  r  r   r  ranger   r  )r   r0  rc  r  r  cache_entryr  	neighborsoffsetneighbor_idneighbor_chunkr   rO  r  prefixs                  r   r  RAGAgent._get_neighbor_context
  si   I<<
+I	8}H 226:I*0b!	VGVaZ0F{"+K#ZZ4N!!%%i0kN4F4FzSU4V4Z4Z[j4kD))$/G!%%j"599&AD{m,FGD6**dNFVdsm34% 1& 9  	I	s   E	 	EEc                    / nSnU H  nU R                  UR                  SS5      5      nUR                  S0 5      R                  SS5      nUR                  SS5      nU(       d  Ma  U R                  U5      n	U	(       a  S	R                  U	S
S 5      nUS
S nSU SUS SU S3n
[	        U
5      nXK-   U:  a    OUR                  U
5        XK-  nM     SR                  U5      $ )z1Fallback context builder using raw chunk content.r   rm  r   rc  r0  rd  r  r   r   NrV  r  r  r  r  r  r  r  )r   rr  rQ  r  r   r  )r   r  rT  r  r  ry  rm  r0  r  r   r  r  s               r   r  RAGAgent.build_raw_context
  s    $F))&**Y*CDGZZ
B/33HiHFJJw,E227;I((9Ra=1etnG$VHM%CyPRSJz?L,z9  ,*N) %, ~~m,,r   rz  c                 *   / nUR                  S5      nU(       a  UR                  SU 35        UR                  S5      nU(       a  UR                  SU 35        UR                  S5      =(       d    / nU(       a1  UR                  S5        USS  H  nUR                  S	U 35        M     UR                  S
5      =(       d    / nU(       a6  SR                  S USS  5       5      nU(       a  UR                  SU 35        SR                  U5      $ )z:Format a stored document digest into concise context text.rl  zTitle: rk  z
Overview: rh  r=  Nr   r  rg  r  c              3   @   #    U  H  oR                  5       v   M     g 7fr$  )rl  )r   r  s     r   r   5RAGAgent.format_digest_for_context.<locals>.<genexpr>*  s     &M*{{}}*r  rV  r>  r  r@  )	r   rz  r  rl  rk  rh  r  rg  themes_displays	            r   r  "RAGAgent.format_digest_for_context  s    

7#LL75'*+::j)LL:hZ01ZZ-3
LL'#BQr%\* ( H%+!YY&M&!*&MMNx'789yyr   r  c                    U(       a  U R                   (       d  / $ UR                  5       n[        [        R                  " SU5      5      n/ nU R                   R                  5        H  u  pxUR                  S5      =(       d    Un	U	R                  5       n
[        U
R                  5       5      nUR                  U5      n[        SXJ5      R                  5       nU(       d  US:  a  M  U R                  X5      nU(       d  M  [        U[        U5      [        S[        U5      5      -  5      nUR                  X45        M     UR                  S SS9  USU  VVs/ s H  u  noPM	     snn$ s  snnf )	z5Find the most relevant theme capsules for a question.r  r  Ng?r   c                     U S   $ r  r  r  s    r   r   7RAGAgent.find_relevant_theme_snippets.<locals>.<lambda>I  s    d1gr   Tr  )r   r   r  r   r   r   rr  r   r  r   rI  compose_theme_snippetrx   r   r  r  )r   r!  r  r  r&  r  r  r  r  r  theme_lowerr  r  rk  r^  r  r  s                    r   r  %RAGAgent.find_relevant_theme_snippets0  s4   t22I!)RZZ
NCD+-"&"5"5";";"=I!++g.;)K%++-Kk//12K$11+>G(~KQQSJzD000GGwJGs1c.>Q7R(RS/0 #>  	-t<*1&5/:/JAw/:::s   E/r  c                    UR                  S5      =(       d    SnUR                  S5      =(       d    / nUR                  S5      =(       d    / nUR                  S5      =(       d    / n/ nU(       a(  UR                  UR                  5        SUS    35        O.U(       a'  UR                  UR                  5        SUS    35        U(       a(  SR                  US	S
 5      nUR                  SU 35        U(       aQ  U V	s/ s H  o(       a  X;  d  M  U	PM     n
n	U
(       d  US	S
 n
UR                  SSR                  U
S	S
 5      -   5        SR                  U5      R	                  5       $ s  sn	f )z/Compose a concise snippet from a theme capsule.r  zRelated Themer  rh  r  r  r   rc  Nr   Key Points: z	Sources: r  r  )rr  r  rl  r  r   )r   r  r  r  r  rh  r  r  joined_pointsr   filtered_sourcess              r   r  RAGAgent.compose_theme_snippetL  s;   kk'*=o;;z*0b[[.4"
++i(.BLLK--/08A;-@ALLK--/0:a=/BC IIj!n5MLL<78+2`7a,!J_7`##*2A; LLtyy1A"11E'FFGyy%%''  as   E.E.c           	      ~   U(       d  g/ nUSS  GH  nUR                  S5      =(       d(    [        US   5      R                  R                  SS5      nUR	                  SU 35        UR                  S	5      nU(       a  UR	                  S
USS  35        UR                  S5      =(       d    / nU R                  USS 5       H  nUR	                  SU 35        M     UR                  S5      =(       d    / nU(       d  M  UR	                  SSR                  USS 5       35        GM     SR                  U5      $ )z/Format distilled insights for prompt inclusion.r   Nrt   rl  r0  r  r   z* rk  z  - Summary: r  rh  r   z  - Key Point: rg  z  - Themes: r  r  )rr  r   r  r   r  r  r  )	r   r  r  r  rl  rk  rh  r  rg  s	            r   r   RAGAgent._format_insight_sectione  s   |GKK(ZD1B,C,H,H,P,PQTVY,ZELL2eW&{{:.H}Xds^,<=> \28bJ..z"1~>ug67 ?[[*0bFv|DIIfRaj,A+BCD $ yyr   analysis_notesentity_hintsc                 \   U R                  U5      nSR                  S U R                  U R                  * S  5       5      =(       d    Sn	SSSSS	.R	                  US
5      n
U R                  U5      SS nSU SU(       a  UOS SU	 SU(       a  UOS SU SU SU(       a  UOS SU
 S3nU$ )z3Assemble the final reflective prompt for the model.r  c              3   ,   #    U  H
  nS U 3v   M     g7fr  r  r  s     r   r   4RAGAgent._build_reflective_prompt.<locals>.<genexpr>  r  r  NzNone.zProvide a concise, evidence-backed answer referencing specific peoples or cultures. Mention uncertainty if evidence is limited.zYExplain the historical narrative step-by-step, citing specific cultures and time periods.zICompare the groups or ideas clearly, noting similarities and differences.zGProvide a succinct overview highlighting the most relevant information.r  z4Answer clearly and cite the evidence from the notes.r  zmYou already drafted internal notes about the user's question. Use them to craft the final answer.

QUESTION: z

ANALYSIS NOTES:
zNo notes available.z

SHORT-TERM MEMORY:
r  r  r  z

EVIDENCE CONTEXT:
r  z7No graph-based relationships identified for this query.z

INSTRUCTIONS:
- a=  
- Follow the ReAct pattern with at most two iterations to reason: alternate between Thought, Action, and Observation.
- Allowed actions: review_notes (inspect analysis notes and distilled insights), consult_entity_graph (use the hints above), synthesize_answer (shape the final response).
- Keep each Thought under two sentences and ground it in evidence.
- Final Answer should include 1 concise paragraph and, when helpful, a short bullet list citing document titles.
- If information is missing, acknowledge it briefly.

OUTPUT FORMAT:
Thought: <internal reasoning grounded in evidence>
Action: <review_notes|consult_entity_graph|synthesize_answer>
Observation: <result of the action>
[Repeat Thought/Action/Observation up to two times if needed]
Final Answer: <concise user-facing response referencing sources>

FINAL ANSWER:)r  r  r|   r}   rr  r   )r   r!  r  r  r  r  r  r  insight_briefr  
guidelinescontext_excerptr   s                r   r  !RAGAgent._build_reflective_prompty  s    44X>"g4;Q;QSWSdSdRdRe;f"ggrkr Yvf`	

 #mS
T 	 --.>?F*  "'< = >     %5 6 7 

     #\ ] ^ , /%L r   r  c                    U(       d  g[         R                  " SU[         R                  [         R                  -  S9nU(       d  UR	                  5       nUS4$ USUR                  5        R	                  5       nUR                  S5      R	                  5       n[         R                  " SUS[         R                  S9nU(       a  US	   R	                  5       nXT4$ )
zASplit the ReAct transcript into reasoning trace and final answer.)r   r   zFinal Answer\s*:\s*(.*)r   r   Nr   z\n(?=Thought\s*:))maxsplitr   r   )r   rk  r   r   r   r  r   r   )r   r  r  rO  reasoning_tracefinal_answertrailing_splits          r   r  )RAGAgent._extract_final_answer_from_react  s    		4l"--Z\ZcZcJcd"((*GB;&~6<<>{{1~++- "6qXZXeXef)!,224L,,r   r  r  c           	      @   / nUR                  SS5      /UR                  S/ 5      -   n[        5       nUR                  S/ 5       HH  n[        R                  " SUR	                  5       5      nUR                  U R                  U5      5        MJ     U H  n	[        R                  " SU	5       H  n
U
R                  5       n
[        U
5      S:  a  M$  [        R                  " SU
R	                  5       5      nU R                  U5      nUR                  X-  5      nU(       d  Mv  [        U5      [        S[        U5      5      -  nUR                  X45        M     M     UR                  S	 S
S9  USU  VVs/ s H  u  nnUPM
     nnnU R                  U5      $ s  snnf )z=Pick sentences from the insight that align with the question.rk  r   rh  rg  r  z(?<=[.!?])\s+r  r   c                     U S   $ r  r  r	  s    r   r   6RAGAgent._extract_relevant_sentences.<locals>.<lambda>  s    !r   Tr  N)rr  r  r   r   r   r  r  r   r   r   r  rx   r  r  r  )r   r  r  r  pooltextsr  r  r  r   rP  sentence_tokensre  r  r  r  r  r   s                     r   r  $RAGAgent._extract_relevant_sentences  sc    )+Z,-L"0MMe[[2.E::j%++-@Lt66|DE / DHH%5t<#>>+x=2%"$**Z9I"J!%!8!8!I(55n6RSGs1c..A'BBU-. =  			nd	3)-fu6gaT	6$$Y// 7s   7Fsent1sent2	thresholdc                    U(       a  U(       d  gX:X  a  g[        U5      [        U5      :  a  UOUn[        U5      [        U5      :  a  UOUn[        U5      S:  a  g[        [        R                  " SUR	                  5       5      5      n[        [        R                  " SUR	                  5       5      5      nU(       a  U(       d  gUR                  U5      nUR                  U5      n	U	(       d  g[        U5      [        U	5      -  n
X:  $ )z<Check if two sentences are very similar (for deduplication).FTrN  r  )r   r  r   r   r   r  r  )r   r  r  r  shorterlongerwords1words2r  r  rk  s              r   _sentences_similarRAGAgent._sentences_similar  s    E > u:E
2%e*s5z1uw<" RZZ
EKKM:;RZZ
EKKM:;V**62V$&U3
&&r   r  c                    U(       d  gUnSn[         R                  " USU[         R                  S9nSn[         R                  " USU[         R                  S9n[         R                  " SSU[         R                  S9n[         R                  " S	U5      n[	        5       n/ nU GH   nUR                  5       nU(       d  M  [         R                  " S
SUR                  5       R                  5       5      n	[         R                  " SSU	5      n	U	R                  5        V
s/ s H  n
[        U
5      S:  d  M  U
PM     nn
SR                  [        [	        U5      5      5      nU(       d  M  X;  d  M  [        U5      S:  d  M  UR                  U5        UR                  U5        GM     U(       a-  SR                  U5      nUR                  S5      (       d  US-  nO&[         R                  " SSU5      R                  5       n[         R                  " SSU5      R                  5       nU(       a*  UR                  S5      (       d  UR                  S5      S-   nU$ s  sn
f )z<Clean up the introduction text, removing redundant patterns.r   z#Themes:\s*([^.]*?)(?:Themes:\s*\1)+z
Themes: \1r   z!Title:\s*([^.]*?)(?:Title:\s*\1)+z	Title: \1z(Key Points:\s*){2,}r  	[.!?]+\s+r  r   r   r   rM  )ru  !r;  ru  )r   r   r   r   r  r   r   r   r  ru  r  r  r  rstrip)r   r  r  theme_patterntitle_patternr   r  r  rP  r  r  r   word_keys                r   r  RAGAgent._clean_intro  s    ?ff]M;bmm\ =ff]L+R]][ ff4nkY[YfYfg HH\;7	u!H~~'H 
B0@0F0F0HIJZ8J !+ 0 0 2A 21c!fqjQ 2EAxxs5z 23HxH0SZ1_ ''1" "" ))$45K''88s" &&k:@@BK ffVS+6<<> {33ODD%,,S1C7K- Bs   I5IrP  c                 N   U(       d  gUR                  5       n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU5      n[        R                  " SSU5      n[        R                  " SS	U5      nU(       aG  UR	                  S
5      (       d1  [        UR                  5       5      S:  a  UR                  S5      S-   n[        UR                  5       5      S:  a  gUR                  5       $ )zEClean up a sentence, removing metadata markers and fixing formatting.r   z'^(Title|Overview|Key Points|Themes):\s*r   z)\s*(Title|Overview|Key Points|Themes):\s*r      ^[-•*]\s*r   z([a-z])([A-Z])r   )ru  r  r;  r  ;r   ru  rN  )r   r   r   r   r  r   r   r  )r   rP  rO  s      r   r  RAGAgent._clean_sentenceD  s    .." &&CRXZXeXef&&EsG[][h[hi &&W5 &&g.&&*Hg> 7++,EFF7==?#a'!..-3 w}}"$}}r   c           
         U(       d  g[         R                  " SSU[         R                  S9n[         R                  " SSU[         R                  S9n/ nSnU HA  nX4-  nUS;   d  M  [	        U5      S:  d  M   UR                  UR                  5       5        SnMC     UR                  5       (       a  UR                  UR                  5       5        [	        U5      S:  aK  [         R                  " SU5       Vs/ s H)  oUR                  5       (       d  M  UR                  5       PM+     nn[        5       n/ nU GH  nU(       a  [	        UR                  5       5      S	:  a  M*  U R                  U5      n	U	(       d  MD  [         R                  " S
SU	R                  5       R                  5       5      n
[         R                  " SSU
5      n
U
R                  5        Vs/ s H  n[	        U5      S:  d  M  UPM     nn[	        U5      S:  a  M  SnU H  n[        UR                  5        Vs/ s H  n[	        U5      S:  d  M  UPM     sn5      n[        U5      n[	        U5      S:X  d  [	        U5      S:X  a  Mi  [	        UR                  U5      5      n[	        UR                  U5      5      nUS:  d  M  UU-  S:  d  M  Sn  O   U(       a  GM  UR                  U	5        UR                  SR                  U5      5        GM     U(       d  U$ / n/ n[        USS 5       H]  u  nnUR                  U5        [	        U5      S:  d  M(  SR                  U5      nUR                  U5        UR                  S5        / nM_     U(       a   UR                  SR                  U5      5        SR                  S U 5       5      R                  5       n[	        U5      S:  a  U$ U(       a  U$ U$ s  snf s  snf s  snf )z7Final formatting pass to clean up any remaining issues.r   z(\b(Key Points|Title|Overview|Themes):\s*r   r  z.!?r   r   r  rN  r  r   r   r   r   Fr   r  TNr  c              3   R   #    U  H  oR                  5       (       d  M  Uv   M     g 7fr$  r  r  s     r   r   0RAGAgent._format_final_answer.<locals>.<genexpr>  s     FID44Ir  r  )r   r   r   	MULTILINEr   r  r   r   r  r  r   r  r  r  r  r  )r   r  r   r  charr   r  r  rP  rO  r  r  r   r  r  r  r  r  r  	formattedr  r  r  ry  s                           r   r  RAGAgent._format_final_answerb  sX    CRWYWdWdeF",,G 	DOGu}w<"$$$W]]_5 G  ==??W]]_- y>A,.HH\6,JX,Jqggi,JIX u!Hs8>>#34r9 **84G 
B0E0E0GHJZ8J !+ 0 0 2A 21c!fqjQ 2EA5zA~ !L!	 Y__->!M->#a&1*!->!MN
 #E
}%*c*o.Bm88DE"=#6#6z#BC!#,(>(D#'L "  < ''0%)I "L  M 	$%5cr%:;KAx) < A%HH\2	  +  $! < SXXl34FIFFLLN v;Mv+V+O Y( B "Ns$   :O+O+O03O0*O5O5c                    ^ U(       d  gUR                  5       R                  5       mT(       d  g/ SQn[        U4S jU 5       5      $ )z*Detect generic or non-informative answers.T)r  z$according to the distilled documentsz"i analyzed the available documentszno relevant documents foundzno documents availabler  c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  )r   signallowereds     r   r   -RAGAgent.is_generic_output.<locals>.<genexpr>  s     C?W$?r(  )r   r   r3   )r   r   generic_signalsr   s      @r   r  RAGAgent.is_generic_output  s=    **,$$&
 C?CCCr   r  c                    [        5       nU H  nUR                  5       n[        U5      S::  d  X@R                  ;   a  M3  UR	                  U5        UR                  S5      (       a%  [        U5      S:  a  UR	                  USS 5        M  UR                  S5      (       d  M  [        U5      S:  d  M  UR	                  USS 5        M     U$ )	z6Remove trivial stop words from a collection of tokens.r   r   r   Nr  esrt   )r  r   r   COMMON_STOP_WORDSr  r  )r   r  filteredr  token_lowers        r   r  RAGAgent._filter_stop_words  s     UE++-K;1$7M7M(MLL%##C((S-=-A["-.%%d++K0@10D["-.  r   r  c                 *  ^ U(       aY  UR                  5       nUR                  5       m[        UR                  5       5      S:  a  [	        U4S jS 5       5      (       d  U$ [        U5      R                  R                  SS5      R                  5       $ )z(Produce a readable title for a document.r   c              3   ,   >#    U  H	  oT;   v   M     g 7fr$  r  )r   r  r   s     r   r   +RAGAgent._humanize_title.<locals>.<genexpr>  s        5D  RCe^  RCr(  )r   r  
permissionrS  r  r   )r   r   r   r   r3   r   r  r   )r   r0  r  rO  r   s       @r   r  RAGAgent._humanize_title  s|    oo'GMMOE7==?#q(  5D  RC  5D  2D  2DF|  ((c288::r   )r   r~   r   ry   r{   rz   r}   ru   rw   r|   r   rv   )r  r   )r   r  )rt   )r   r$  )NNr  )rV  )N  )r   )r  )Nr   )r   )g333333?)Z__name__
__module____qualname____firstlineno____doc__r|  r  r   r   r   r  r  rs  r   r   r   r   r  rD  rJ  r   rQ  rY  r_  r   r   r  r   r  r  r  r   rr  ry  r}  r  r  r  r  r  r  r  rd  r  r  r8  r  r+  r,  r  rF  rI  re  r.  r/  r}  r0  r   r   r  r   r  r  r  r  r  r&  rB  r  r  r  r  r  r   r  r  r  r  r   r  r  r  r  r  r  r  r   r  r  __static_attributes__r  r   r   r   r      s
   G	N 
	D&%  Y^ il S S (@3 @3 @D/'# /'% /'b /c  /c  /D  @@s @3 @5 @# $s) 	 	C 	QT 	_b 	3  UX Q'C Q'PTUYZ]_bZbUcPd Q'l -  d38n!56-  -  
d38n		- ^^^ tCH~&^ 	^
 
^@]3 ]$tCQTH~BV ]gj ]or ]> :>	_ _  _  !d38n!56	_ 
 
_ Bh-T(1*2f# 2f 2f 2fr ,0=A<< < 	<
 $C=< %T$sCx.%9:< 
<|LUc LUS LUX[ LUlo LU  DG LU  LO LU\W|W| W| 	W|
 W| W| !c3h0W| 
W|r*s *c *TW *hk *ps *XLS L3 Ls LWZ L*<c <3 <_b <gj <|
# 
4 
$sDd38n9L4M/M*N >EDc3h,@ Ehk Etxy}  B  DG  G  zH  uI EN d38n  S#X  SV  0 48	$$ sCx.$ !0	$
 
c3h$LS c $s) 6 48	>> S>> !0	>
 
$sCx.	!>@&
 &
c3h8P &
P[Dd3SRUXCV>W9W4X []a [2PT#tCcSVhDW?X:X5Y P^b P*@C @T#s(^8L @QYZ^_bdg_gZhQi @D=B4T#tCQTH~BU=V8V3W =B\` =B~tCc3h,?'@ (T#tCH~*=%> *d3i DI "T$sDcN/B*C TlCc3h'(  	
 #  
*%/ %/S	 %/N4.3 4.S 4.QT 4.j/c /c /8%S %U %> :>	K-K- K- !d38n!56	K-
 
K-Z9v$tTWY\T\~J^A_ O O Ob`-x} `-Dc htCH~6N B%C %4S> %SV %_cdg_h %N-C -: S#X  3  2;S ;QTUXQYHZ ;jm ;vz{~v ;8(T#s(^ (8TWX[T\K] (il (2 T#s(^0D    (<< < tCH~&	<
 < < < < 
<|-S -U3PS8_ -0 	0C0 c3h0 	0
 
c0@' 'C 'E 'UY '@7# 7# 7r  <b,3 b,3 b,HDhsm D D"# 3s8 ;c ;hsm ; ;r   r   )(torchr   r  r+  pathlibr   r   collectionsr   	itertoolsr   concurrent.futuresr   r   airagagent.configr	   r
   r   r   r   r   r   r   r   typingr   r   r   r   r   r   r   difflibr   	rapidfuzzr   airagagent.grok_apir   r   r   r  r   r   <module>r!     sR     	     $ " ?
 
 
 C B B #  HP3; P3;r   