ANCi8GddlZddlZddlZddlmZddlmZddlmZddl m Z ddl m Z m Z mZmZddlmZmZmZddlmZmZmZmZmZdd lmZmZmZGd d Zd Ze d k(reyy)N)Path) PDFProcessor) VectorStore)RAGAgent)PDF_DIR DOCUMENTS_DIRDEFAULT_RETRIEVAL_KMIN_CHUNK_SCORE)enhance_retrieved_contextsynthesize_answerstrip_reasoning)RAGSystemErrorVectorStoreError ModelErrorDocumentProcessingErrorValidationError) get_loggerlog_performancelog_error_with_contextc>eZdZdZdZdZdefdZd dZdZ d Z y) RAGSystemctt|_t|_t |_d|_d|_y)NF) r__name__loggerr pdf_processorr vector_store rag_agent _initializedselfs (/var/www/html/leadgen/airagagent/main.py__init__zRAGSystem.__init__s1 * )^'M!c<|jt|_yy)z6Initialize RAG agent (called on demand to save memory)N)rrrs r!initialize_modelzRAGSystem.initialize_models >> !%ZDN "r#c |jjdtj} |jj |j |j |jj}|r|jjdt|d|j|jj||jj|jjdt|d|jjrt|jjnd}tj|z }t|jd|||r t|nd|jjd |d |S#t$r }t!|j|d d id d}~wwxYw)zInitialize the RAG systemz!RAG System initialization startedz Processing z new document chunkszSuccessfully processed 0 raw chunks (enrichment available via UI button)r system_setup)total_documentsnew_chunks_processedz)RAG System initialized successfully with z documents operation setup_systemzFailed to initialize RAG systemN)rinfotimerload_existing_indexensure_enriched_indexsync_chunk_files_to_indexrprocess_all_newlenr% add_documents save_index documentsr Exceptionr)r start_time new_chunks total_docssetup_durationes r!r,zRAGSystem.setup_system!s <=YY[ "     1 1 3  & & (  * * ,++;;=J   ;s:.??S!TU%%'!!// ;!!,,.   #:3z?:KK{!|}=A=N=N=X=XT..889^_J!YY[:5N DKK*4BLs:RS U KK  H T^_ `   "4;;K3PRs t  sFF22 G;GGreturnc  tttjd}|syt }|j j xsgD];}|jdixsijd}|s+|j|=g}d}|D]} t|dd5}tj|} dddt tr| sD| d} d} t| tr8| jdixsijdxs| jd} | s| |vr| D cgc]H} t| ts| jd s&t| jditsG| J} } | s|j| |j| |d z }|r`|j j#d t%|d |d |j j'||j j)|S#1swYXxYw#t$rYwxYwcc} w#t$r(}|j j+d|Yd}~yd}~wwxYw)z Ensure every stored *_chunks.json file is represented in the vector index. Returns number of sources added. *_chunks.jsonrmetadatasourcerzutf-8)encodingNcontentz Sync: adding z chunks from z# missing source(s) into FAISS indexz-Sync: failed to sync chunk files into index: )listrrglobsetrr6getaddopenjsonloadr7 isinstancedictextendrr-r3r4r5warning)r chunk_filesexisting_sourcesdocsrc chunks_to_add sources_added chunk_filefdatafirstrAcvalidr<s r!r1z#RAGSystem.sync_chunk_files_to_indexKsh 4 tM277HIK"u ((228b .wwz2.4"99(C$((- . MM) # j#@,A#yy|, "$-TQeT*#ii B7=2BB8LcPUPYPYZbPcF--%)BqJq$,?AEE)DTYcdedidijtvxdy{ZABB$$U+ $$V," 9 #<   =]1C0DMR_Q`aD"EF!!// >!!,,. C,, $B  KK  "OPQs S T s)H5AH5;H5H 'H=H A.H53H0 H0 H0<H0BH5H H  H-)H5,H--H55 I&>I!!I&Nctj}|t|ddnd}|jjd||rt t|dkDrdnd|t }|s |jj ddgd St|ts,|jj d t|d gd S|j}|s |jj d d gd S|jjs |jj ddgd S |j |jj||}|jj!dt |d|jjdt |dt#|dddD]\}}|j%dd} |j%dd} |j%dd} |j%dij%dd } |jjd!|d"| dd#d$| d%d&| d%d'| d%d( t'j(d)|j+D chc]} t | d*kDr| c} d+t,ffd, }g}|D]<}|j%ddt.k\}||d-k\}|s|s,|j1|>|rt|jjd.t |d/t.d(|dd0Dcgc]}|j%ddd%}}|jjd1||}nl|rj|jj d2t.d3|dd*Dcgc]}|j%ddd%}}|jj d4||dd*}|s&|jjd;|dddt#|dd0dD]M\}} |jjd?|d"| j%dd dd#d@| j%ddd%Od} |jjdAt=||}t?||}tA|}|rdC|xsdvrd|jjdD|j8jC||E}|j8jE|||}|r tA|n|}tj|z }tG|jdF|t |t ||r t |ndGH|jjdI|dJdK||dLdMS#t$r8}t|j|d|ddddt|gd cYd}~Sd}~wwxYwcc} wcc}wcc}w#t2$rD}|jj d5|j4d6|j4gd cYd}~Sd}~wt6$r9}t|j|d7|dddd8|j4gd cYd}~Sd}~wt$r8}t|j|d9|dddd:t|gd cYd}~Sd}~wwxYw#t$r+}|jj dB|d}Yd}~6d}~wwxYw#t$r8}t|j|dN|ddddOt|gd cYd}~Sd}~wwxYw)Pz1Search for relevant documents and generate answerNdNonezProcessing question: z...zEmpty question providedzPlease provide a question.)answersourceszInvalid question type: zQuestion must be a string.zQuestion is only whitespacez%Please provide a meaningful question.z+Search attempted but no documents availablezANo documents available in the system. Please add some PDFs first.model_initialization)r+questionzError initializing model: )kzFound z search resultsz Retrieved z chunks with scores: rEscoreg vector_score keyword_scorer@rAUnknownz Chunk : 2z - Combined: .3fz (Vector: z , Keyword: )\b\w+\br=cx|jdixsijddxsd}|sytjddt|tj}|j ddj d dj }ttjd |}t|zS) Nr@rArarz \.(pdf|txt)$)flags_ -rp) rIresubstr IGNORECASEreplacelowerrHfindallr3) candidaterUcleaned src_tokens query_tokenss r!source_overlapz3RAGSystem.search_and_answer..source_overlaps }}Z4:??"MSQS&&"c#hbmmT!//#s3;;CEKKM J!@A <*455r#z Filtered to z high-quality chunks (score >= zFiltered chunk scores: zAll chunks below threshold z, using top 3 anywayzTop 3 scores: zQuery validation failed: zInvalid search query: vector_searchzVector store error: searchzUnexpected search error: z*No relevant documents found for question: z.No relevant documents found for your question.zGenerated answer with z sourcesz Source z - Score: z%Applying grounded answer synthesis...zSynthesis failed: z$No sufficiently relevant informationz5Falling back to extraction-based answer generation...)requestion_answerr)question_length results_count answer_lengthz"Successfully answered question in z.2fsT)rbrcenhancedanswer_generationzError generating answer: )$r.ryrr-r3r rQrNtypestriprr6r%r7rrdebug enumeraterIrwr}r|intr appendrmessagerrprepare_source_summariesr r r format_contextgenerate_answerr)r rerfr8question_previewr<search_resultsiresultrhrirjrAtrfiltered_resultsrBscore_ok title_match score_list top3_scoresrc final_answerenhanced_contextsynthesized_answercontextrbdurationrs @r!search_and_answerzRAGSystem.search_and_answersYY[ 2:2F3x=#.F 01A0BHY\]`ai]jYknqYq5wyBz{| 9#A KK   9 :6 (C( KK  "9$x.9I J K6 >># KK   = >A    ** KK   M N]     ! ! #@ !..55h!5DN KK  s>':&;?K L KK  z#n*=)>>RS T&~cr':A> S 6 7C0%zz.#> & ?C @ J377)L   8A3b ]5QT+U_`lmp_qq|~KLO}PPQ"RS  S::j(..2BCq6A:L  6S 6 " # /55#./A,Q/14 {$++A.  /     <4D0E/FFefuevvw!xyDTUWVWDXYqw!4S 9:Y Y   #::,!GH!1 ##&A/ARRf$ghESTVUVEWX!%%"5c!:;X X ##n[M$BC!/!3( KK  I(SWTW.IYZ [J  * nn==nMG KK  5c'l^8L M&wr{A6 B 6   9QCr&**Xy2QRUSU2V1WWabhblblmtvybz{~a"AB B L $   !HI#Z:@v6f yy{Z/H DKK):H*-h-(+N(;=IL(9q R KK  A(3qQ R&"  c  "4;;AWemnrores3t u6s1vh?  ..Z Y  KK  ";AII; G H2199+>    "4;;^fgkhk^l3m n0 <   "4;;W_`dadWe3f g5c!fX>  8 $ ##&8$<=#  $0  "4;;ATbjkolobp3q r5c!fX>  s2V2D3X6W6 >X A XW;2AXX%X*B$\ >[ C$\ 2 W3;-W.(W3.W36X [9Y [ [.Z [ [-[ [ [ \ \=\ \\ ] -]] ] c`|jjrt|jjdkDri} t|jjdkDr)|jjdjdi}|jds|jdry|j jdt|jjdytttjd }|sytd  |j|y#tt f$r*}|j jd|i}Yd}~d}~wwxYw#t$r}td |j d}~wt"$r}td |d}~wwxYw) zMEnsure the vector store uses enriched knowledge cards rather than raw chunks.rr@z"Could not access sample metadata: Nsummarycard_idzVector store has zB documents (not enriched). Skipping enrichment for faster startup.r?z0Rebuilding knowledge index from stored chunks...zVector store rebuild failed: z!Unexpected error during rebuild: )rr6r3rI IndexErrorAttributeErrorrrr-rFrrrGprint!_rebuild_vector_store_from_chunksrrr7)r sample_metadatar<rRs r!r0zRAGSystem.ensure_enriched_index-s    & &3t/@/@/J/J+Ka+O!O %t((223a7&*&7&7&A&A!&D&H&HUW&XO""9-1D1DY1O   #4S9J9J9T9T5U4VWY"Z[4 .33ODE   @A   2 2; ?'/ % !!$Fqc"JK"$ %(   1!))= >   5aS9 :  s<A D/E+/E(> E##E(+ F-4F  F-F((F-c`g}|D]N} t|d5}tj|}t|tr|j |dddP|s tdytdt|d t|_ |j|rN|jj||jjtdt|d yy#1swYxYw#t $r}td|d|Yd}~d}~wwxYw#t$r}td |jd}~wt $r}td |jd}~wt"$r}td |jd}~wt $r}td |d}~wwxYw)zLRebuild the vector store using knowledge cards generated from stored chunks.rBNz#Warning: Failed to load chunk file rlz'No chunk data available for rebuilding.u • Loaded z stored chunksu! • Rebuilt vector store with r'u • Vector store error: u • Model error: u" • Document processing error: u1 • Unexpected error rebuilding vector store: )rKrLrMrNrFrPr7rr3rrr%r4r5rrrr)r rR all_chunksrXrYrZr<s r!rz+RAGSystem._rebuild_vector_store_from_chunksSs % OJ O*c*0a99Q?  + D   ! ! #!!// ;!!,,.9#j/9JJz{|%00 O;J?@! (!  . !  ! GA3-  !s*+C.(C.+BC..D$D$ DD$__main__)!rLr.rwpathlibrrrrrmistral_integrationrconfigrrr r answer_synthesisr r r exceptionsrrrrrlogging_configrrrrrrrr#r!rs` &$(OOZZONffP &!P zFr#