Qi$ddlmZmZmZmZmZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z mZddlmZddlmZddlZddlZddlmZddlmZmZmZmZmZmZmZmZm Z m!Z!m"Z"ddl#m$Z$m%Z%m&Z&dd l'm(Z(m)Z)dd l*m+Z+ee,Z-e.ee-j.d <d e-j.d <e/ej`jcddZ2e$e,Z3eZ4e jjZ6eedz dz Z7dZ8dZ9e!Z:e/ej`jcddZ;e/ej`jcddZd8g3d?Zbe-jd@dAZce-jdBdCZde-jdDdEZee-jdFd8g3dGZfe-jdHdIZge-jdJd2g3dKZhdcdLZidgdMZjdNZke-jdOd8g3dPZle-jdQd2g3dRZme-jdSdTd2g3dUZne-jdVdTd2g3dWZoe-jdXd8g3dYZpe-jdZd2g3d[Zqe,d\k(r_erd]erd^ej`jcd_djjd`vZue-jeeeuayy)h)Flaskrequestjsonifyrender_templatesend_from_directoryN)ThreadPoolExecutor as_completedsecure_filenamePath) RAGSystem) PDF_DIR PROCESSED_DIRWEB_HOSTWEB_PORT NOTES_DIR DOCUMENTS_DIR DIGESTS_DIRENRICHMENT_DEFAULT_THREADSENRICHMENT_MAX_THREADSENRICHMENT_STATUS_PATHENRICHMENT_LOG_PATH) get_loggerlog_performancelog_error_with_context)FileUploadErrorValidationError)RAGAgent UPLOAD_FOLDERi MAX_CONTENT_LENGTHMAX_CONCURRENT_UPLOADSmetadatazprocessing_status.jsonii,)ENRICHMENT_STATUS_STALE_THRESHOLD_SECONDSi#ENRICHMENT_STATUS_HEARTBEAT_SECONDSFct5tr dddydatjtd}|j t jddddy#1swYyxYw)z9Start the processing worker thread if not already runningNTtargetdaemonz Processing worker thread started)processing_worker_lockprocessing_worker_running threadingThreadprocessing_workerstart app_loggerinfo) worker_threads '/var/www/html/leadgen/airagagent/app.pystart_processing_workerr6>s` < $ <<%)!!((0A$O :; <<E,  e # u( 1 1 6 . 3 3 5  6s)CCCCCC76C7 force_resetreturnc ltj}t5tjditjdttj dxstj dxs|}tj dxsi}|r? t t|t d|jDt|}t d|t|z }t|dtd <|r2tjd d d d d i||d t dddytj ddk(rE|tkDrz,cleanup_enrichment_status..s%XQquuY27a8Xs)+)defaultgr]rrrzEnrichment status cleared)rmrnrorrrrprNTrmrunningrz$Enrichment stalled (no activity for zs). Please restart enrichment.)rmrorrprF)rErrrrrr=maxrvaluesrBrvrtsave_enrichment_statusr%int)rrxrrrs r5cleanup_enrichment_statusrs ))+C )$$^R8$$^5OP)--o>iBSBWBWXaBbifi (,,^<B   #-(X,BUBUBWXbghubvw! Cu]';!;< -2=!-D/*   $ $ 6#$ "!$ &  # $=))@   *i 7MLuFAF*5AF* F'$F*&F''F**F3c& tjjddttd5}t j t |ddddy#1swYyxYw#t$r"}tjd|Yd}~yd}~wwxYw)zSave enrichment status to fileTrYr\r]r^Nz"Failed to save enrichment status: ) rr`rarMrNrbrrBr2rCrUs r5rr rdrec tj|d|} tjjddt5t td5}|j tj|dzddddddy#1swYxYw#1swYyxYw#t$r"}tjd|Yd}~yd}~wwxYw)z2Append enrichment events to log file for auditing.) timestampeventTrYa Nz&Failed to write enrichment log entry: ) rErr`raenrichment_log_lockrMwriterNdumpsrBr2debug) event_typepayloadrzrVexcs r5log_enrichment_eventrsYY[   E I""(((E  2)3/ 21 5)D01 2 2 2 2 2 2 2 IA#GHHIsM'B$B(B ;BB$ B BB!B$!B$$ C-C  Cc  |tjdt}t5tj |||||t j t j |dt dddy#1swYyxYw)zUpdate enrichment statusNr)rmrnrorrrprr)rr=rrrrtrEr)rmrnrorrrs r5update_enrichment_statusr$sr(,,^=WX  !   .&yy{!YY[( "    ! ! !s A A99Bct5tjdi}|j|i}|t||d<|t||d<|r||d<t j|d<t jtd<|r!tj ddk7r dtd<t dddy#1swYyxYw) Nrchunk total_chunksrorprrmr)rrrrrrEr=r) file_name chunk_index chunk_totalroactiverzs r5update_enrichment_file_activityr5s !"--nbA!!)R0  " -E'N  "$' $4E. ! &E) 99;i-1YY[/* '++H5B*3 h ' !!!s B/B??Cct5tjdi}||vr7|j|dt jtd<t dddy#1swYyxYwNrr)rrrrrrEr)rrs r5clear_enrichment_file_activityrFsY %"--nbA   JJy$ '15 o . " $ %%%s AA""A+ct5itd<tjtd<tdddy#1swYyxYwr)rrrrErrr5clear_all_enrichment_activityrNs< !,..)-1YY[/* !!!s /?Achttdstt_tjS)z5Get or create a thread-local RAGAgent for enrichment. rag_agent)hasattrthread_local_agentrrrrr5get_thread_rag_agentrYs$ %{ 3'/z$  ' ''rc t|}t dt |t S#ttf$r tcYSwxYw)z.Clamp requested thread count to allowed range.r;)r TypeError ValueErrorrrminr)r|counts r5sanitize_thread_countr_sC*E  q#e34 55 z "*))*s '??c^|rd|vry|jdddj}|dvS)z"Check if file extension is allowed.Fr;)pdftxt)rsplitlower)rlexts r5 allowed_filergs8 s(* //#q !! $ * * ,C .  rct5tdtjDcdddS#1swYyxYw)z5Return the number of files currently being processed.c3JK|]}|jddk(sdyw)rmrr;N)r=)rrzs r5rz-get_active_processing_jobs..qs!d (@SWc@c1ds##N)rrsumrPrrrr5get_active_processing_jobsrns6 ed"3":":"<ddeees$5>/c.tdttS)Nz index.html)enrichment_default_threadsenrichment_max_threads)rrrrrr5indexrss #=5 rz/uploadPOST)methodsc  tjtjxsd tj ddtj vr'tj dtddidfStj d}|jdk(r'tj d tdd idfS|rt|js4tj d |jd tdd idfStt}|tk\r6d|d}tj dd|t||tddfS|jdtj|j!|jdt"j$d}|kDr1tj ddtdd|dzdidfSt'|jtj(j+t"j$dtj(j-}|rcd} t/d5}t1j2|j5j7}ddd|jd|j5}t1j2|j7} |jd|| k(rtj dd d} t>5tAtBd!rgtBjDrWtAtBjDd"r=tBjDjFr#tItBjDjF} dddtd$d%d&| d'd(fStj d)d*dd+l&m'} | } | jQ} | vrtStTd,z d-z }|j-r_ t/|d.5}tWjX|}dddvr|=t/|d/5}tWjZ||d01dddtStTd"z tSj\d2z }|j-r|j_tj d3d4d5|jafd6}tcd7dd8d9|jd:;tdjg|titj d<d=tdjktj d<d>td$d?d7dd@dAfS#1swYxYw#t8t:f$r%}tj=d|d}Yd}~*d}~wwxYw#1swY@xYw#tJ$r#}tj=d#|Yd}~jd}~wwxYw#1swYxYw#1swYxYw#YxYw#tJ$rR}tjz }tmt|dB|dCdDtddEto|idFfcYd}~Sd}~wwxYw)GNunknownzFile upload request from filez&Upload request missing file part from rCz No file partrfz(Upload request with empty filename from zNo selected filezInvalid file type attempt: z from zrd?} ntj%d@d } d d d | 2td45}d5 |d6fi}| j/| |d;C} dD} d d d | j0d=k(r| j3}|j5d>r|j5dEr|dE}tjdFt|dG dH}g}t7d t||D]>}||||z}|j9s|j;|| t|dI| dJdK@|rtjdLt|dM t tdz t jdz }|j<j?dIdIN t|dO5}d d l}|j@||dPQd d d tjdRt|dS|t ddUdV| dWt|dXdYt|| dZn%tj%d[ d\| jB|rt|d k(r,tjd^ d_t dd d`ddaiy tjdbt|dM t ddcddt|dedft|dgtjdht|dit ddjdkddlitEtFdmrtFjHs(tjdnt dd doddpiy tFjKtjdqt|dit ddrdst|dtdut|dg tL5tFjHjO|tFjHjQd d d d } tEtFjHdr=tFjHjRr#ttFjHjR}t-j,!z }tWtdz| |{tjd| d}t|d~tY|dPdt dd7dt|dtY|dPddt|tY|dPdy #1swYxYw#t$rJ}tjd|t dd dt|dt|dYd }~y d }~wwxYw#t$r$}tj%dA|dBYd }~Pd }~wwxYw#1swYZxYw#1swY3xYw#1swYxYw#t$r#}tj%dT|Yd }~d }~wwxYw#t$r&}tj%d] d\|Yd }~d }~wwxYw#1swY xYw#t$rJ}tjdv|t dd dwt|dxt|dYd }~y d }~wwxYw#t$r#}tjUdy|Yd }~Fd }~wwxYw#t$rD}t|}t[t|d dd t dd d|d|dYd }~y d }~wwxYw)z*Process uploaded file in background threadr zStarting file processing...stage initializingrwz*[Background] Step 1: Extracting text from r#zExtracting text from PDF... extract_textrrNz[Background] File z+ already processed, loading existing chunksr'zLoading existing chunks...loading_existing_chunksrrrJz[Background] Loaded z existing chunks for z-[Background] Failed to load existing chunks: rz Failed to load existing chunks: load_existing_chunks_failed)rrCz, was marked as processed but no chunks foundz,File marked as processed but no chunks foundmissing_chunks.pdfz*[Background] Normal extraction failed for z, trying OCR fallback...z%Text extraction failed, trying OCR... ocr_fallbackz[Background] Text file z4 produced no chunks - file may be empty or too shortzOFile is empty or contains insufficient content (minimum ~100 characters needed)text_too_short)r file_typezhttp://127.0.0.1:5003/ocrzhttp://127.0.0.1:5002/ocrrag__zStarting OCR (job: z)... ocr_starting)r ocr_job_idrrzapplication/pdfd)job_id max_pagesz%[Background] Trying DeepSeek-OCR for z (job_id: i0*)filesdatar:rsuccessDeepSeekz5[Background] DeepSeek-OCR failed, trying Tesseract...z![Background] DeepSeek-OCR error: z, trying Tesseract...)rr: Tesseracttextzackground] OCR extracted z chars from iT)ocr ocr_service)contentrsourcechunk_idr$z[Background] OCR created z chunks from rYr\r]r^z[Background] Saved z OCR chunks to z,[Background] Failed to save chunks to file: (zOCR (z ) extracted z chunks. Processing... ocr_complete)r chunk_countrz[Background] OCR failed for rz$[Background] OCR fallback error for z)[Background] Failed to extract text from z& even with OCR. File may be corrupted.z7Failed to extract text. File may be corrupted or empty.extract_text_failedz[Background] Step 2: Extracted 2z Extracted z" chunks. Adding to vector store... chunks_ready)rr z[Background] Step 3: Adding z raw chunks to vector storer8z Adding chunks to vector store...vector_store_preparerz)[Background] Vector store not initializedVector store not initializedvector_store_missingz[Background] Step 4: Adding UzAdding z chunks to vector store...vector_store_writez2[Background] Error adding chunks to vector store: zError adding to vector store: vector_store_errorz+[Background] Could not get document count: file_upload_processing)rlrirrjz3[Background] Successfully processed uploaded file: rz chunks, zs)rz Successfully processed! Created z chunks in s)rr duration_secondsfile_processing) operationrlrjz Error processing uploaded file: zError: ).r}r2r3 pdf_processorr process_pdfr rstemrLrMrOlenrBrCstrsuffixrrTrequestsryrrEpost status_coderNr=rangestripappendr`rarbrr rag_systemrinitialize_modelvector_store_lock add_documents save_indexrrrrvr)"r processorchunks chunk_filerVrNrHfile_extis_pdfr5 deepseek_url tesseract_url ocr_responseocr_service_usedrpdf_filer ocr_data_formds_errorocr_dataocr_text chunk_sizei chunk_text save_error ocr_error doc_countduration error_msgrj file_pathrirl start_times" r5process_file_asyncz'upload_file..process_file_asyncsq ^ ( 1"N3 "LXJ WX( 1"N3 7(N "..tI?>OO&8 Bm$no, $4&(AB "&m!4{!BXH[H[G\\hEi!iJ!((* #!%j#!66!)216&OO.B3v;-Odemdn,op#((+=hZGs)tu0$$J#*,<"= V!1#H~44::+?qTYY[AQ@R%S 1$(1*TB,:*"U ")T2 xh%+hBS-T$UE7APS,TM x *2WX`Waaklvkww{0| }/7}}\QV]jty}/z #/#;#;s#B|GXGXGZG^G^_hGi7A$4$.$6$69n$p37L x(/!%i!6?()/(HFW1X(Y/7}}]RWaf}/g 3> 0? (33s:'3'8'8':H'||I68<<;O+3F+; *2McRZm_\hiqhr0s t.2 )+).q#h-)L !+A19!AjL1IJ'1'7'7'9(. 7A4>6>8;F @DUe8f 7*)+ !+$*$.OO6OPSTZP[}\ijris4t$u26m1D{1RX\]eXfXkXkWllxUy1yJ$.$5$5$;$;DSW$;$X%x-1*c-B)Ka,7,5DIIfa,J)K)3:McRXk]Zijtiu8v(w%=(0(4(**/0@/AcRXk]Zp(q8FWZ[aWbtD/E %&'..1MhZWYZfZkZkYl/mn "S[A%5"((+TU]T^_E*FG0$$U#*,A"B "A#f+m\d[e fg(  V -OP$23v;O ">s6{mKf gh( 6"$:; z>:*BYBY$$%PQ,  6&(>? ++-">s6{mKf gh( c&k]*DE$8VU *="//==fE"//::<= Xz66 DI`I`IjIj$' (?(?(I(I$J  99;3 ,Dh'/(1.7(1 3 "UV^U__abeflbmannwx}GIJyKxLLN!OP(6s6{m;uU]_`OaNbbcd!,'*6{,1(A,> C66 ) #&,,/\]^\_-`a4 ( ( !"B3q6( K0MX[\]X^&_ # #`$-x * 2 25VW_V``u3v w wx x x ??@)K)K,5%x(2(:(:=ijtiu;v(w(w%x%k"**-QRZQ[[]^g]h+ijjkv==! $$'YZ[Y\%]^,  8QA(b>b)d. Ac? B d.^ ^ _!A_d._!!d.$ `-` ` ```a*!`+&a*.`83`;; a'a"a*"a''a** b3bd.bd.b&!b)) c<2Ac72d.7c<<d.? d+d& d.&d++d.. e;7:e66e;rz%File uploaded, starting processing...queued)rrhrirjrkrFile z( added to processing queue, queue size: z( saved, processing started in backgroundz4" uploaded successfully. Processing in background...)rormrlrirn file_uploadr.rjrTz#Unexpected error during file uploadzUpload failed: )8rEr remote_addrr2r3rrTrrlrrrr"seekosSEEK_ENDtellappconfigr pathjoinrLrMhashlibmd5read hexdigestrRrSrr=rr;rrr2rBr/rget_processed_filesr rrNrOrbr1unlinksaver}r<putr6rArr3)r active_jobsromax_size file_exists existing_hashrVrHnew_file_content new_file_hashrSrr@processed_files metadata_filer$rBrXrTrjrVrirlrWs @@@@@r5 upload_filerx{sHJ##0yIvC3I;?@  &   !G {S TG^45s: :}}V$ ==B    !I)U VG%7893> > < 6   !:CZCZ&z'>'> LQ[QhQhQrQr,/ 0G0G0Q0Q,R S!'z1jk!%'0  ";H:EY Z[6(N "+"?"?"A.$($7*$DG]$]M$++-!!%mS!98Q+/99Q<8'83$,X$6!%mS!9AQ $ (Aa @A "&m!4{!BXH[H[G\\hEi!iJ!((*"))+-zI;mI;WX )` ` F !   3!(&&!%   /0!%z)QRbRhRhRjQklm %z)QRSz)]^" "     e FFW% %  #J1#!NO $  %SS!K$$'EaS%IJJK(88AA! T C99;+ 'iX V 1 ?3q6(!;<=sBBCsAX2AX2 A X2AX2'B X21A*X2 V(2VV"A5X2W!A8WW!X23AX2 X*X5X*X)X*1DX2VVW,W X2 WX2WW!! X *XX2X  X2XX*X'"X**X/,X22 Z ;AZZ Z z/askc tj}tjjdtjxsd} tj dxsi}d|vrt ddidfSt|jddj}|st dd idfSttd rtjs: td tj}dt_ td |d tj|}t!|trt |gddfSt |dfS#t$r3}td|t ddt|idfcYd}~Sd}~wwxYw#t$rR}tj|z }t#t$|d||ddt ddt|idfcYd}~Sd}~wwxYw)NzX-Forwarded-ForrfTsilentquestionrCzQuestion is requiredrzQuestion cannot be empty _initializedz'Initializing RAG system for question...zSystem ready with z documentszError initializing RAG system: zSystem initialization failed: r^)answersourcesrquestion_processingr]z)Unexpected error in ask_question endpointzError processing question: )rErheadersr=r_get_jsonrr3r9rr;r}print setup_systemrBsearch_and_answer isinstancerr2) rWrjrr|rSinit_eresultrHrTs r5 ask_questionr8sJ##$5w7J7J7PbQI#Ot,2 T !G%;<=sB Btxx B/0668G%?@A3F Fz>2*:Q:Q _?@&335 *. '*9+Z@A --h7 fc "f<=sB B6?C' ' _7x@A+I#f+)WXY[^^^ _ O99;+ /iU] ^ 7 #>s1vh!GHI3NNOsa,F 59F / F 9E 4F > F F(F<F=F FF G%AG G% G%z/statusGETc P d}d} ttdrtjrt5ttdrgtjrWttjdr=tjj r#t tjj }tjdu}dddno ttdz }|dz }|jrGt|d5}tj|}t|tr t |nd}dddtt'j(d }tt'j(d }t+||zd d } g} i} ttdz dz }|jr+ t|d5}tj|} ddd| D]} | j,| j/j0t3| j/j0dz d| j/j4| j,| v| j,| vr,| j7| j,ij7ddndd} | j9| t;|t |t |t | || Dcgc]}|j,c}| ddfS#1swYxYw#1swYxYw#YxYw#t $r#}t"j%d |Yd}~d}~wwxYw#1swYixYw#YqxYwcc}w#t $rR}t"j=d|ddl}|j@t;ddtC|idfcYd}~Sd}~wwxYw)NrFr}rr embeddingszdocuments_metadata.jsonrJzCould not get doc count: *.pdf*.txtc6|jjSN)statst_mtime)xs r5zget_status..s@Q@QrT)r{reverser$rrr]r )namesizesize_mbuploaded processedrA)r pdf_files txt_filesr model_loaded file_listrrzStatus check failed: rCr^)"rr;r}r=rrr2rr rrLrMrNrOrrrBr2rrglobsortedrrst_sizervrr=r:rrC traceback print_excr3)rSrembeddings_dirrwrVr$rHrr all_filesrrvrV file_infors r5 get_statusrasJDI   z>2z7N7N&Dz>:z?V?V":#:#:KHZMdMdMnMn(+J,C,C,M,M(NI#-#7#7t#CL DD%)-%8<%GN$25N$NM$++-!-5['+yy|H9CHd9SH YZI[g./ g./ 9y06Q[_`  ]+j8;QQ    ! --3&*iilO3 # (I!!(00 !1!9!9[!I1M%NN,55&^^>[d[i[im|[|/--innbAEEmUVWCD I   Y ' ((YYy>(*34Q!&&4    ]DD[[    8< =  33 &5 I045 #8Q!ABCSHH IsM &LB K7 L3K943K,'K9/A7M ' L=3L0 L=C/M M M K)$L,K61K99K>;L L- L("M (L--M 0L:5L==M? M N%AN N% N%z/reranker-statusc ddlm}t|dfS#t$rtddddfcYSt$r#}tdt |ddfcYd}~Sd}~wwxYw) z!Get status of the reranker moduler)get_reranker_statusrFzReranker module not available)enabledrCr^N)rerankerrr ImportErrorrBr3) _get_statusrHs r5rrshA?{}%s** Z53RSTVYYY A53q6:;S@@AsA A AA A z /ocr-statuscDddl} i}g} |jdd}|jdk(r|j} |jd d}|jdk(r |jjd g}t |t|||jd d k(rd ndddfS#t$r}t |dd}Yd}~d}~wwxYw#YYxYw#t$r:}tjd|t dt |idfcYd}~Sd}~wwxYw)zCGet OCR service status and active jobs - for RAG system integrationrNzhttp://127.0.0.1:5003/healthr9r unavailable)rCrmzhttp://127.0.0.1:5003/jobsjobsrmokzOCR service is readyzOCR service unavailable)rrprrozOCR status check failed: rCr^) r5r=r7rNrBr3rr2r2rC)r5 ocr_healthocr_jobs health_resprH jobs_resps r5get_ocr_statusrs7/  D",,'Eq,QK&&#-(--/   %A1 MI$$+$>>+//;%x=1;1IT1Q-Wp      D#&q6]CJ D   /4QC89Q()3../sXC2B1AC0C1 C:C C CCCC D%/DDDz/ocr-progress/c@ddl} |jd|d}|jdk(rt|j dfStd|dd fS#t $r>}t jd |d |tt||dd fcYd}~Sd}~wwxYw) zDGet progress of a specific OCR job - proxies to internal OCR servicerNzhttp://127.0.0.1:5003/progress/rr9rz Job not found)rCrzOCR progress check failed for rr^) r5r=r7rrNrBr2rCr3)rr5resprHs r5get_ocr_progressrsA||=fXFPQ|R   s "499;', ,_GH#M M A9&A3GHQ6:;S@@As#?AA B3BBBz/static/ctd|S)Nstatic)r)rls r5 serve_staticrs x 22rz/pdf/cddlm}||}t|z }|jr6|jj dk(rt tt|StddidfS)zServe PDF files for viewingrr rrCz PDF not foundr) werkzeug.utilsr rrLr4rrr3r)rlr rypdf_paths r5 serve_pdfrs_/#H-M&HX__224>"3w<?? G_- . 33rz/chunks/cddlm}ddl}ddlm}||}|t dz ||j dz }|jrC t|d5}|j|}dddtd|t|d d fStd didfS#1swY3xYw#t$r%}td d t|id fcYd}~Sd}~wwxYw)z"Get chunks for a specific PDF filerr Nr rrrJT)rrlrrArrCzError reading chunks: r^zChunks not found for this filer)rr rNpathlibr rr1rLrMrOrr2rBr3) rlr rNr ryrBrVrArHs r5 get_chunksrs/#H-Mm${2]8K8P8P7QQ]5^^J Nj#& &!"1 &) #F     !ABCSHH & & NG'=c!fX%FGH#M M Ns6 B,B ."B, B)%B,, C5CCCz/processing-statusctttj}|iid}t5tj D]U\}}t |||}|jddk(r ||d|<1||jd|z tksN||d|<W dddt|d|d <t|d|d <t|d fS#1swY8xYw) z9Return snapshot of all processing jobs (active + recent).)rrrecent)rxrmrrrrN active_count recent_countr) rWrrErrrPrurr=#PROCESSING_STATUS_RETENTION_SECONDSr2r)rxsnapshotrlrzrs r5list_processing_statusesr s ))+CH >0668 >OHe3HeMJ~~h'<7/9"8, s;;?bb3=HX&x0  >> #8H#56H^"8H#56H^ 8 c !!>>sA"C CC#z"/processing-status/c t|}tttj |}|rt t ||dfSttdz dz }|jrU t|d5}tj|}ddd|vr%t ddd||j d d d dfS t d d d ddfS#1swYDxYw#Y!xYw)z Get processing status for a filerr$rrJNrrzFile already processedr r)rmrnrorArzNo processing status found)rmrnror) r rWrrPr=rrr rrLrMrNrO)rlryrzrwrVrvs r5get_processing_statusr#s$H-M  ! !- 0E 1-GH#MM]+j8;QQ    ! --3&*iilO3 O3""-$'#;"1-"@"D"D]TU"V $  43     33 s$4 C%C0C%C"C%%C)z/enrichc | tjdxsi}|jd}t|}t t t jddk(rStddt jddt jd d t jd td d fStjt|fd}|jtdd|ddfS#t$r=}tjd|tddt!|idfcYd}~Sd}~wwxYw)z%Trigger enrichment for new files onlyTrzthreadsrmrz>Enrichment is already running. Please wait for it to complete.rnrrorfr)rCrmrnrori)r*argsr+z0Enrichment started. Processing new files only...r)rormrr[zError starting enrichment: rCzFailed to start enrichment: r^N)rrr=rrrrrrr.r/enrich_documents_asyncr1rBr2rCr3)rrequested_threadsrenrichment_threadrHs r5enrich_documentsrFsDPt,2 HHY/,->?   !#   *i 7Y#-11*a@,00B? 1 5 5nF` a    &,,4JR^Q`imn!I(      P6qc:;#?Ax!HIJCOOPs$B4C57=C55 D;>2D60D;6D;c ~ |t}t|} td|tddddd|t t j d}t|}|dk(rtdd d dd|td d yi}tjr|d?|d@}n| |z }|| k(rdA}n dB|dC|dD}tdd || | |tMtd || ||Et.jOdF|dG| dH|dIy#YxYw#1swYgxYw#YoxYwcc} w#1swY/xYw#t8$r>} t.j1d"| td dd#t;| dd|Yd} ~ yd} ~ wwxYw#t8$r[} t.j1d%| j"d&| | j"d'd%| j"d&t;| ddd(}Yd} ~ d} ~ wwxYw#t8$rYwxYwcc}w#t8$rC} t.j1d9|d-d&| td:|d-t;| ;Yd} ~ d} ~ wwxYw#tC|d-d|d/<wxYw#t8$r^} t.j1dJ| tdKt;| LtMtd ddMt;| dd|Yd} ~ yd} ~ wwxYw)NzMEnrich documents in background - only process files without cards (threaded).N run_started)rrrz*Checking for files that need enrichment...rrrzNo PDF files found run_completedzNo files to processroz *_cards.json_cardsrfTr$rrJz%All files already have enriched cardszAll files already enriched files_queued)rrzFound z file(s) to enrich. Starting (z thread(s))...r}rz)Vector store not available for enrichmentrr%z2Failed to initialize vector store for enrichment: z$Vector store initialization failed: )rError enriching rFrrroenriched_chunksr r;Pro Finished rrrc3XK|]"}t|jdxsg$yw)rN)r2r=)rrJs r5rz)enrich_documents_async..s$$cQS/@)A)GR%H$cs(*Zz-Updating vector store with enriched chunks...)r1zIndexing enriched chunks vector_update)rrArzIndexed enriched chunks for zVector store update failed for vector_update_failedrrC_z5No enriched chunks generated. Vector store unchanged.zEnrichment complete! Processed z file(s) with z total chunks.zNEnrichment completed but no files generated new cards. Check logs for details.z"Enrichment completed with issues. z succeeded, z skipped.)enrichedtotalrArozEnrichment completed: rz files enriched, z total chunkszError in enrichment process: run_failed)rCzEnrichment failed: )(rrrrrrrr2rrLr1replacer rrMrNrOrr:rr;r}rr2rCr=load_existing_indexr<rBr3process_file_for_enrichmentrr=rr enumerater%apply_enriched_chunks_to_vector_storerr3)rrrexisting_cards card_file source_stemfiles_to_enrichrvrwrVrI file_stemtotal_files_to_processrHresultsrrrnrJenriched_resultstotal_chunks_processedenriched_countidxro skipped_counts r5rrhs1 (6Lnb]LA A/[]^`acopg./ )n !  $[#7KQPQS_ ` :O P     &^^N;  "+.."8"82"FK26N;/ ]+j8;QQ    ! --3&*iilO3 " 1H I}}/I^4S&&x0 1  $[#7^`kmx{G H :V W !$_!5^O3TqAFF3TU  +,,J<.Xf g "   z>2*:Q:Q z>:*BYBY$$%PQ,Xq:XZ[]^`lm'3++??A//1.2J+3( H 4XLY NN6 " q O12H HBNNOH $ 9%D9X]]O)D&   .vf~>; D(/`!!%% 2BquuM^G_A``!$$cRb$c!c-.  $?&   ))9C 5 V53F6NLfg9&.&QbJcd(vf~VYZ`arZsZywyVz{"2s3C/D)D(J#JKH,! 6vf~6FG'.$ 36&>B04F,-) 5, %G&   A 77G~VlUmm{|G2^CM 66j>~>Nl[hZiirs c7B04F,-2 b8<=\Q8%' 10CCF8.LaQRT`aa bsoA(W>.W-!Q9+W: RRR$WW 2W<WR $?W$AR(*R(0>R.R(6 W S2 AW(U6W<U)U)%U))AW.A6U.$BW9Q>;WR RR WR% R(( S/14S*%W*S//W2 U;AU WUW U&"W%U&&W. V:78V5/V=5V::V==WW X<AX77X<c  jddddd} tjdtjdz }|j sFd jd |d <t d j tj|d |St |d5}tj|}dddrt|tsFdjd|d <t dj tj|d |St|}tjd|d|dt dj||t}tj fd}tj|d} | j!fd} |j#|| |} j%| r|j'd| t| djdt| ddtjt| t| d t d!jt| "|Sd#jd|d <tj|d t d$j  |S#1swYxYw# j%wxYw#t($rr} d%jd&t+| |d <t+| |d'<tj|d t d(jt+| )Yd} ~ |Sd} ~ wwxYw)*zWorker function to build enriched chunks for a single PDF. Args: pdf_file: Path object for the PDF file thread_count: Number of threads to use for chunk-level parallel processing FrfNrrzLoading chunksrrzNo chunks found for z , skipping.rofile_missing_chunks)rrJz Invalid or empty chunk data for rfile_invalid_chunksz&Generating knowledge cards (starting, z thread(s))rrro file_started)rr rcjts- tjjts,yy#t$rY#wxYwr)waitr&rrrB)rI stop_eventsr5 heartbeatz.process_file_for_enrichment..heartbeatCsD oo&IJ3HMMB!oo&IJ!sA AATr)c Ftj||d|d|dy)NzGenerating knowledge cards (r)r)rr)currentrrIs r5progress_callbackz6process_file_for_enrichment..progress_callbackOs* + #!6wiqqI  r)rrrrz enriched chunks))rrr rozCards ready for indexingfile_completed)rrz!No enriched chunks generated for file_no_cardsrrrC file_errorr)rrrr1rLrrMrNrOrrr2rr.Eventr/r1build_enriched_chunkssetrtrBr3)rIrrrBrV chunks_datar agentrheartbeat_threadrrrHrs` @r5rrs  FFM' ?OP" l%CC   ""6x}}o[ QF9  !6X]] K +HMM6)CT UM *c " 'a))A,K '*[$"?"B8==/QR SF9  !6X]] K +HMM6)CT UM+& ' 1R]jPQ]P^^ihj k^(--[gst$&__&  %++9TJ   #99+YjzF9GO NN   MM#2"?3&x}}oRO8L7MM^_   , 002   !!1 WZ[jWk l M#DHMM?RS TF9  +HMM6)CT U x}} E M} ' 'L NN & M.x}}oRAxHya&w' vi?PQ\ SVLL M Ms_BK K%J!;AKBK-J.BKAK!J+&K.KK L> A'L99L>c|stjd|dy tjd|dt|dt5t j j||t j jdddtjd|y#1swY"xYw#t$r!}tjd|d |d}~wwxYw) z Safely replace existing vector store entries for a source with enriched chunks. This method uses replace_source_documents which rebuilds the index to ensure consistency and prevent index corruption (index/document count mismatch). z No enriched chunks provided for z, skipping vector store updateNzUpdating vector store for z with z enriched chunksz&Successfully updated vector store for z Error updating vector store for r) r2rTr3r2r=r;rreplace_source_documentsr?rBrC) source_namerrHs r5rrts =k]Jhij4[MOH\G]]mno  1  # # < <[/ Z  # # . . 0 1 @ NO  1 1 ;K=1#NO s/+B5 ?B) B5)B2.B55 C>CCz/enrichment-statusctttjddk(r6tjdr!tjdi}|r dtd<t tdfS)zGet current enrichment statusrmrrrr)rrrr=r)rs r5get_enrichment_statusrsbX&&05F5J5J>5Z(,,^R@ *3 h ' $ %s **rz/enrichment-resetcXttdtdditdfS)z/Force-clear enrichment status (admin recovery).T)rrr)rrrrrrr5reset_enrichment_statusrs-$/ It9'89 :C ??rz/delete/DELETEc4 t|}t|z }|jstddidfStj d|t 5 ttdrftjrVtjj|tjjtj d|dd d d t|d z t|d z t |d z g}g}|D]j}|js |j#|j%t'|j(tj d |j(lt+t,dz dz }|jrx t/|d5}t1j2|} d d d | vrI| |=t/|d5}t1j4| |dd d d tj d|d |j#|j%|tj d|tdd|d|dd fS#t$r#}tjd|Yd }~d }~wwxYw#1swYxYw#t$r&}tjd|d|Yd }~d }~wwxYw#1swYxYw#1swYxYw#t$r"}tjd|Yd }~d }~wwxYw#t$r=}tj7d|tddt'|idfcYd }~Sd }~wwxYw#t$r=}tj7d!|tdd"t'|idfcYd }~Sd }~wwxYw)#z7Delete a PDF/TXT file and all associated processed datarCzFile not foundrzDeleting file: rzRemoved z from vector storez"Error removing from vector store: Nr _cards.json _digest.jsonzDeleted processed file: Error deleting rr$rrJr\r]r^z from processed files metadatazError updating metadata: zDeleted source file: zError deleting source file: zFailed to delete source file: r^TrZz and associated data deleted)rro deleted_filesrzError deleting file: zFailed to delete file: )r rrLrr2r3r=rr;r_remove_source_from_indexr?rBrTrrrrmr:r3rr rrMrNrOrbrC) rlryrVrHprocessed_pathsr proc_pathrwrVrvs r5 delete_filer!sCK'1 m+ !G%567< </-9: M M:~6:;R;R++EEmT++668OOh}o=O$PQ  M }o\: : =/5 5 ]O<8 8   ( KI!K$$&!((Y^^)<=OO&>y~~>N$OP  K]+j8;QQ    ! D--3&*iilO3 O3' 6mS1@Q /1Q?@OOh}o=[$\]  V       / OO3M?C D }o-IJ*     W M""%Gs#KLL M  M M.!K&& 2aS'IJJK33@@ D""%>qc#BCC D V   ;A3? @G'Ec!fX%NOPRUU U V K045#:3q6(!CDEsJJKs2MMJA6I AMAJ%'M KK/K K#!K9L>M I?I:4J:I??JJ M J>J93M9J>>MK KKK L#L;MLM M2M MM MM N2N NNz /delete-allc  tjdttjd}ttjd}||z}d}g}|D]} t |j }t5 ttdr/tjrtjj|dddt|d z t|d z t |d z g}|D]#} | j#s| j%%|j%|j'||d z }t)t*dz dz } | j#r, t-| d5} t/j0i| dddt5 ttdrtjrgtj_gtj_gtj_tjj8Ltjj8j:} t=j>| tj_tjjAtjddddtCdd|d||ddfS#t$r&}tjd|d|Yd}~d}~wwxYw#1swYxYw#t$r0}tjd |j d|Yd}~d}~wwxYw#1swYxYw#t$r#}tjd|Yd}~d}~wwxYw#t$r"}tjd|Yd}~d}~wwxYw#1swYxYw#t$r=}tjEd|tCddtG|idfcYd}~Sd}~wwxYw)z3Delete all PDFs/TXTs and rebuild empty vector storez.Deleting all files and rebuilding vector storerrrrzError removing z from vector store: Nrrrr;rrr$rr\zError clearing metadata: zRebuilt empty vector storezError rebuilding vector store: TzDeleted z% files and rebuilt empty vector store)rror deleted_countrzError deleting all files: rCzFailed to delete files: r^)$r2r3rrrr rr=rr;rrrBrTrrrrLrmr:r rrMrNrbrsearchable_texts_rawsearchable_texts_lowerrdfaiss IndexFlatIPr?rrCr3) rrrr#rrVryrHrr rwrV dimensions r5delete_all_filesr*sLLHIg./ g./  )   # LI L /  ? 'ee":~>:CZCZ&33MMm\e"}o\$BB=/ ==]O<"@@# "1+I '')!((*+   "$$]3" 5 L@]+j8;QQ    ! D--%IIb!$%  J J:~6:;R;R8:J++5CEJ++@EGJ++B!..44@$.$;$;$A$A$C$C 8=8I8I)8T //5++668OO$@A J!-0UV**     ]%e"**_]OK_`a_b+cdde ee. L""_Y^^4DBqc#JKK L%% D""%>qc#BCC D J""%DQC#HII J J J* L5aS9:#;CF8!DEFKKLs AN"K'=K??J(>AK'?8K'7'N L0+L#L0 NN C8M N( K1K KKKK$ K'' L 0%LNL  N#L-(L00 M9MNMN N (NN N  N  NN O #2OO O z/reprocessing-statusc ttdz dz }|jr8 t|d5}t j |}dddt dfSt ddd d dddd dfS#1swY+xYw#tjttf$r"}tjd|Yd}~\d}~wwxYw) zGet current reprocessing statusr$zreprocessing_status.jsonrJNrz$Failed to load reprocessing status: rrzNo reprocessing in progressrf)rmrnro current_filerrchunks_created) r rrLrMrNrOrrQrRrSr2r) status_filerVrmrHs r5get_reprocessing_statusr/=s}% 25OOK Ik3' &11 &6?C' ' 0     & &$$gw7 I   CA3G H H Is. A?A3 A?3A<8A??B?B::B?z/extract-entitiesc tj}|rd|vrtddddfSddl}|d}igidd}gd }|D]K}|j ||j }|s&|j d j|d d <ngd }g}|D]&} | |j vs|j| (|r||d<gd} gd} |j t fd| Drd|d<nt fd| Drd|d<nd|d<tjd|td|dddS#t$r;} tjd| tdt| ddfcYd} ~ Sd} ~ wwxYw)z Extract user entities from conversation using simple pattern matching POST body: { "user_message": "I am a project manager...", "assistant_response": "Great! I can help..." (optional) } user_messageFzuser_message required)rrCrrN) demographics interestssurvey_preferencesexpertise_level)z1I (?:am|work as) (?:a |an )?([a-z\s]+?)(?:\.|,|$)z%I\'m (?:a |an )?([a-z\s]+?)(?:\.|,|$)z3(?:my job|profession|role) is ([a-z\s]+?)(?:\.|,|$)r;r2 profession) cannabisbusinessfinance marketing compliancewellnesshealthgrowingretail cultivationr3)expert experiencedsenioradvancedzyears of)znew tobeginnerz just startedlearningc3&K|]}|v ywrrrphrase message_lowers r5rz,extract_entities_endpoint..sD6v&DrAr5c3&K|]}|v ywrrrHs r5rz,extract_entities_endpoint..sHV=(HrKrE intermediatezExtracted entities: Tsimple_patternsg333333?)rentitiesmethod confidencezEntity extraction failed: r^)rrrresearchrgroupr9r:anyr2r3rBrCr3)rrRr1rOprofession_patternspatternmatchinterest_keywordsfound_interestskeywordexpert_phrasesbeginner_phrasesrHrJs @r5extract_entities_endpointr^VsH!~T1 0  N+ "$#    + GIIg|'9'9';Q9M9M9O(6    ( 0G,,,..&&w/ 0 $3H[ !UM$**, D^D D*2H& ' H7GH H*4H& '*8H& '.xj9: '     5aS9:V    s0)E;E)AE/BE F 0F;FF__main__zStarting Flask application...z/RAG system will be initialized on first request RAG_DEBUG)1trueyeson)hostportr)rrfNr)F)rrfrrN)NNN)r;)wflaskrrrrrrarErNrhr.r>concurrent.futuresrr rr rr rr'mainrrerrrrrrrrrrrlogging_configrrr exceptionsrrmistral_integrationr__name__rdr3renvironr=r"r2r;localrrKrrrr%r&rPQueuer<r-Lockr,r6r0rrrrr=rWrcr}rrrboolrrrrrrrrrrrrouterrxrrrrrrrrrrrrrrrrr!r*r/r^rr9rrrunrrr5rus8OO  ?*     ON7( Ho"'l ?#3  RZZ^^,DbIJ ! [ $Y__&m,z9