i dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl m Z mZddlmZddlmZddlmZddlmZmZmZmZmZdd lmZmZmZmZdd l m!Z!eGd d Z"d ddddddddddddddddddddddddddddddZ#gdZ$gdZ%gdZ&gdZ'gdZ(d d!d"d#d$d%d&d!d'd$id(d)d*d+d,d-d%id.d"id"d!d/d0d1id2d2d3d+d4gd5Z)gd6gd7gd8d9d:gd;gdZ*hd?Z+hd@Z,hdAZ-hdBZ.ej^dCej`ej^dDej`ej^dEej`ej^dFej`ej^dGej`ej^dHej`ej^dIej`gZ1dJZ2dKZ3 ddl4Z4dLZ5 ddNl7m8Z9dLZ: ddOl;mZ?dLZ@ ddPlAmBZBdLZC ddlDZDdLZE ddlFZFdLZGGdQdRZHGdSdTZIGdUdVZJGdWdXZKGdYdZZLeeMZNd[eNjd\<e"ZOeIeOZPeJeOZQeKeOZReLeOZSiZTd]dd^ddddddeUejjd_d`da aXe jZZe jZ[e jZ]da^eOjdbz Z`dcZaddZbejjdedfZcdgZdeedheQjebdiegdjegdkeUdlegfdmZhdiegdnedoegfdpZieNjdqdrZkeNjdsdtgudvZleNjdwdtgudxZmeNjdydzguedd{ZneNjd|dtgud}ZoeNjd~dzgueddZpeNjddzgueddZqeNjddtgudZreNjddtgudZseNjddtgudZteNjddtgudZueNjdddzgueddZveNjddzgueddZweNjddzgueddZxeNjddtgudZyeNjddzgueddZzeMdk(rneUejjddZ{eede{eedeEdeCde5eede|eQjeNjde{dMyy#e6$rdMZ5YwxYw#e6$rdMZ:YwxYw#e6$rdMZ=YwxYw#e6$rdMZ@YwxYw#e6$rdMZCYwxYw#e6$rdMZEYwxYw#e6$rdMZGYwxYw)z Advanced Sports Betting RAG Service Incorporates sophisticated PDF processing, FAISS vector search, BM25 hybrid scoring, and DeepSeek/Tesseract OCR fallback. Port: 5001 (configurable via RAG_PORT) NCounter)datetimetimezone)Pathwraps) dataclass)ListDictAnyOptionalTuple)Flaskrequestjsonifysend_from_directory)secure_filenamecdeZdZUdZeej j Zeed<dZ eed<dZ eed<dZ eed<dZ eed<dZ eed<d Zeed <d Zeed <d Zeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed <d!Zeed"<d#Zy)$ RAGConfigzRAG Service Configurationbase_dirNdata_dir cache_dirpdf_dirsource_ranking_configquery_expansion_config chunk_sizechunk_min_size@chunk_max_sizesentence_overlap min_sentences?hybrid_vector_weighthybrid_keyword_weightg?hybrid_bm25_weightmax_chunks_per_source default_kxmin_vector_candidatesvector_oversample_factorhttp://127.0.0.1:5003/ocrdeepseek_ocr_urlhttp://127.0.0.1:5002/ocrtesseract_ocr_urlc|j;ttjj d|j dz |_|j |jdz |_|j|jdz |_|j|j dz dz |_|j|j dz dz |_ |j jdd |jjdd y) NDATA_DIRdata rag_cachecsvconfigzrag_source_ranking.jsonzrag_query_expansions.jsonT)parentsexist_ok) rrosenvirongetrrrrrmkdirselfs @/var/www/html/eventheodds/python_service/advanced_rag_service.py __post_init__zRAGConfig.__post_init__@s ==   DMMF> !!]][8DN << ==50DL  % % -)-)AD])]D &  & & .*.--(*BE`*`D ' TD9 4$7) __name__ __module__ __qualname____doc__r__file__parentr__annotations__rrrrrrintr r"r$r&r(floatr)r*r+r-r/r1r3strr5rErFrDrr!s#(^**11Hd1HdItGT"&4&#'D'JNCNCcM3#'%&#'5' $$!"3"Is!$3$$&c&8c78s88rFr rr#)rr r"r$r&rir%iixr!) technicalresearchlegalsportsdefault)\ moneylinespreadoverunderparlayteaserpropfuturesoddsjuicevigvigorishlinez point spreadtotalhandicapfavoriteunderdogpushcoveratszagainst the spreadbankrollunitroizreturn on investmentedgeevzexpected valuesharpsquarepublicsteamzreverse line movementrlmnbanflmlbnhlncaacollege basketballfootballbaseballhockeybetwagerstakeriskpayoutwinlossprofit sportsbook bookmakerbookiebettinggambling handicappingmodelsystemstrategy analytics statisticsr8trendsinjurylineuprotationrestz back to backtravelschedulehomeawayroadneutralvenuecourtfieldquarterhalfperiodinningovertimelivezin-play closing linez opening linez line movementmarketprice)the_logic_of_sports_bettingsharp_sports_bettingsquares_n_sharpsthe_perfect_betthe_oddsgamblbetting_history)z\bmlb\br moneyballsaberm bill_james curve_ball mvp_machine winshares)z_games_part\d+z_players_part\d+ _standings_teams_records) probabilityrmodelingrresearch_methodsall_of_statisticsthe_art_of_statistics ףp= ?{Gz?皙?g)\(?)rrpropsgeneric_mlb_propsQ?gQ?333333?gg{GzgQ)rrr)elitestrongrpenaltyrr)rrrgQg ףp= ǿg333333ӿ)rrr reference data_dump) tier_weights tag_weights source_rules)zsports bettingrcrz market pricerzedge expected value) player propszprop betzplayer prop linesz alt lines)z mlb baseballzpitcher batterzlineup bullpenzweather park factor) betting_core props_core baseball_corerrr)rrr) named_groupsintent_default_groups sport_aliasessport_default_groups query_rules>rsrrqrrrgrcralinesrrrrrrpricesrmarketsrirr player propr>home run stolen basesr{rbihitswindwalksbatterhitterrrbattersbullpenhittersinningslineupspitcherweatherrpitchers strikeout strikeouts home runs total bases> rrrarrrrrrrr> r:logsarchivedatasethistoryresultsarchivesbacktestdatasetstracking historical backtestingz \bcopyright\bz\btable of contents\bz\ball rights reserved\bz\bintentionally left blank\bz\blimit of liability\bz\bdisclaimer of warranty\bz\bisbn(?:-1[03])?\b?g?TF extract_text) PdfReader)SentenceTransformerc TeZdZdZdZd dedededeefdZ dedededeefdZ y ) LLMServicez0LLM service using Grok Fast Reasoning (Enforced)cBtjjdxs tjjddjjdjd|_d|_d|_d|_|js td yy) N GROK_API_KEY XAI_API_KEY"'z$https://api.x.ai/v1/chat/completionszgrok-4-fast-reasoningFz<[RAG] WARNING: GROK_API_KEY not set. LLM features will fail.) r>r?r@strip grok_api_keygrok_url grok_modeldeepseek_availableprintrBs rD__init__zLLMService.__init__!szZZ^^N;`rzz~~m]_?`ggioopstzz{~> 1"'  P Q!rFprompt max_tokens temperaturereturnc(|j|||S)zGenerate text using Grok API) _call_grok)rCrrrs rDgeneratezLLMService.generate+svz;??rFc :tr |js tdy td|jdt j |j d|jdd|jd|d g||d d d }|jdk(r3|j}d|vrUt|ddkDrD|ddddStd|jd|jr|jddndy#t$r}td|Yd}~yd}~wwxYw)z Call Grok APIz*[RAG] Grok API not configured (no API key)Nz[RAG] Calling Grok (z)...zBearer zapplication/json) Authorizationz Content-Typeuser)rolecontentF)rmessagesrrstream-)headersjsontimeoutchoicesrmessagerz[RAG] Grok API error: z - rUz(empty)z[RAG] Grok API exception: ) HAS_REQUESTSr rr requestspostr  status_coder!lentext Exception)rCrrrresponser8es rDrzLLMService._call_grok/s:4#4#4 > ? 4 ((9> ?}} '.t/@/@.A%B$6 "__*0V!D E",#.# H ##s*}}$T)_)=)A ?1-i8CC.x/C/C.DC_g_l_l VZWZH[r{G|}~ 4 .qc2 3 3 4sB!C96C99 DDDN)rRr') rGrHrIrJrrPrNrOrrrrQrFrDrrs^:R@s@@@X`adXe@!!#!E!hWZm!rFrc eZdZdZdefdZdedefdZdedefdZ ded edefd Z dede efd Z dede e eeeffd Zdded ede eeeffdZdede e dede efdZdedede efdZdededefdZddedede efdZy) PDFProcessorz4Advanced PDF processor with structure-aware chunkingr;c$||_gd|_y)N)ucopyright\s+©?\s*\d{4}zall rights reservedzpage\s+\d+\s+of\s+\d+z^\s*$z^\d+$z [a-f0-9]{32,}z ^[\s\W]*$)r;boilerplate_patternsrCr;s rDrzPDFProcessor.__init__Vs % !rF file_pathrc|jjdk(rhtr |j|St r tt|Str. t|}djd|jDSy|jjd k(r) t|d d 5}|jcdddSy#t$r}t d|Yd}~d}~wwxYw#t$r}t d|Yd}~d}~wwxYw#t$r}t d|Yd}~yd}~wwxYw#1swYyxYw#t$r8t|d d 5}|jcdddcYS#1swYYyxYwwxYw)z(Extract text using best available method.pdfzPyMuPDF failed: Nzpdfminer failed:  c3DK|]}|jxsdyw)rNr).0pages rD z,PDFProcessor.extract_text..ws!$X4T%6%6%8%>B%>$Xs zPyPDF2 failed: r.txtrutf-8encodingzlatin-1)suffixlower HAS_PYMUPDF_extract_with_pymupdfr,r HAS_PDFMINERpdfminer_extractrP HAS_PYPDF2rjoinpagesopenreadUnicodeDecodeError)rCr4r.readerfs rDrzPDFProcessor.extract_textbs    ! ! #v -255i@@ 3+C N;; 1&y1F99$X6<<$XXX    # # % / $)S7;$q668$$ 9!2,QC0112!3-aS1223!1OA3/001$  & $)S9=$668$$$  $sC C1,D$E2D9 E C.C))C.1 D:D  D D6D11D69E>EEFE8, F8F =FFc4tj|}g} |D]E}|jdtjtjz}|j |G |j dj|S#|j wxYw)z&High-fidelity extraction using PyMuPDFr+flagsr7)fitzrJget_textTEXT_DEHYPHENATETEXT_PRESERVE_WHITESPACEappendcloserH)rCr4docrIr:r+s rDrDz"PDFProcessor._extract_with_pymupdfsii "  #}}V43H3H4KhKh3h}i T" # IIKyy IIKs A BBr+filenamec|j|j}gd}tfd|Ddk\rygd}tfd|Ddk\rygd}tfd |Ddk\ry y ) z)Detect document type for optimal chunking)rrcr\r[r_rrc3,K|] }|vsd ywrTNrQr9ind text_lowers rDr;z4PDFProcessor.detect_document_type..sCS 1BqC r#rY) specificationrVapiprotocol algorithmc3,K|] }|vsd ywr\rQr]s rDr;z4PDFProcessor.detect_document_type..sASsj/@qAr`rV)abstract methodologyr conclusion referencesc3,K|] }|vsd ywr\rQr]s rDr;z4PDFProcessor.detect_document_type..sES3*3DqEr`rWrZ)rBsum)rCr+rYfilename_lowersports_indicatorstech_indicatorsresearch_indicatorsr_s @rDdetect_document_typez!PDFProcessor.detect_document_types{ZZ\ !)o C-C Cq HY AOA AQ Fa E/E E JrFc|sgSd}tj||}|Dcgc]/}t|jdkDs |j1c}Scc}w)zSplit text into sentencesz?(?<=[.!?])\s+(?=[A-Z])|(?<=[.!?])\s*$|(?<=[.!?"\'])\s+(?=[A-Z])r,)resplitr*r )rCr+sentence_endings sentencesss rDsplit_into_sentencesz!PDFProcessor.split_into_sentencessOI^HH-t4 #,DaAGGI0C DDDs !AAc  g}|jd}t|D]\}}|j}|rt|dkr'|j r1t|dk\r#t|dkr|j ||dfht jd|}|std|jdd D}|jd} t| dkDs|j | |t|df|S) zDetect headings in documentr7r%PrTz^\s*(\d+)(?:\.(\d+))?\s+(.+)$c3&K|] }|sd ywr\rQ)r9gs rDr;z/PDFProcessor.detect_headings..sH!aAHsNr#) rs enumerater r*isupperrVrrmatchrkgroupsgroupmin) rCr+headingsrirg line_strippednumbered_matchdepth heading_texts rDdetect_headingszPDFProcessor.detect_headingss 4  ' FGAt JJLM C $6$:$$&3}+=+Bs=GY\^G^1 56 XX&F VNH~'<'<'>r'BHH-33A6 |$q(OO\1c%m$DE! F$rFdoc_typec&tj|td}|d}|d}|d}|d}|d}|rt|j|krgS|j |} | r|j || |} | r| S|j ||S)zStructure-aware chunkingrZrr r"r$r&)DOCUMENT_TYPE_CONFIGSr@r*r r_chunk_by_sections_sentence_based_chunking) rCr+rr;r chunk_min chunk_maxoverlapr&rchunkss rD chunk_textzPDFProcessor.chunk_texts&**85J95UVL) +, +, +,/ s4::<(94I''- ,,T8VDF ,,T6::rFrcg}|jd}t|D]\}\}}} |dzt|kr ||dzdn t|} dj||dz| j } | sYt| |dk\sk|j | |} | D] } || d<| | d<|j | |S)zChunk text by detected sectionsr7rTr section section_level)rsr}r*rHr rextend)rCr+rr;rrrr heading_poslevelend_possection_contentsection_chunkschunks rDrzPDFProcessor._chunk_by_sectionss 4 6?x5H . 1A1 k5,-ECM,Ahq1uoa(s5zG"iikAog(FGMMOO3#76BR;S#S!%!>!>PV!W+3E'3E)$-2E/*3 n- . rFc|j|}t||dkrgSg}g}d}|D]k}t|dz}||z|dkDr|rdj|j} t| |dk\r'|j | t|t| dt |dt|} | dkDr|| d ng}t d |D}|j |||z }||d k\st||dk\sdj|j} t| |dk\r'|j | t|t| dt |dt|} | dkDr|| d ng}t d |D}n|rWdj|j} t| |dk\r'|j | t|t| d|S) z$Sentence-based chunking with overlapr&rrTr" r )rsentence_count char_countr$Nc38K|]}t|dzywr\r*r9rvs rDr;z8PDFProcessor._sentence_based_chunking..$GASVaZ$Grc38K|]}t|dzywr\rrs rDr;z8PDFProcessor._sentence_based_chunking..rr)rwr*rHr rVrrk) rCr+r;rur current_chunkcurrent_lengthsentencesentence_lengthr overlap_counts rDrz%PDFProcessor._sentence_based_chunkings--d3 y>F?3 3I ! HH!(ma/O/&9I2JJ} XXm4::< z?f-=&>>MM#-*-m*<&)*o#!$F+=$>M@R S BORSBS }no >Y[ !$$G$G!G   * o -N !55#m:LPVWfPg:g XXm4::< z?f-=&>>MM#-*-m*<&)*o# !$F+=$>M@R S BORSBS }no >Y[ !$$G$G!G? HD -0668J:&)9":: )&)-&8"%j/  rFrc|jdd}t||jddkry|j}|jD])}t j ||tj s)yt jd|}t|dkry|r tt|t|z nd}|d kryy ) zValidate chunk qualityrrr rF\b\w+\br333333?T) r@r*rBr2rrsearch IGNORECASEfindallset)rCrr;r content_lowerpatternwords unique_ratios rDis_valid_chunkzPDFProcessor.is_valid_chunk-s))Ir* w<&**%5s; ;  00 Gyy-?   :}5 u:>8=s3u:U3! # rFN source_namec| |j}|j|}|rt|jdkrgS|j ||}t j |t d}|j||}g}t|D]F\}} |j| |s|| d<|| d<t|| d<|| d<|j| H|S)z.Process a file and return chunks with metadatadrZsourcechunk_id total_chunks document_type) namerr*r rprr@rr}rrV) rCr4rr+rr; raw_chunksrrrs rD process_filezPDFProcessor.process_fileEs  #..K  +s4::<(3.I,,T;?&**85J95UV__T84 !*- %HAu""5&1"-h$%j!(+Jn%)1o& e$  % rF)rZN)rGrHrIrJrrrrPrrDrpr rwrrNrr r rdictrrboolrrrQrFrDr0r0Ss:>  y  #d#s#J  t    ,EEcECDsC}1E,F2;s;c;$tCQTH~BV;.sd5k4TXY]T^&5S5$54:5nD$40dT rFr0ceZdZdZdefdZdedeee ffdZ dedeee ffdZ dZ d ede jfd Zd eede jfd Zdefd ZdZdZdedefdZd edeedefdZdededeee ffdZdeeeee ffdeee ffdZdeeeee ffdeeeee fffdZdedeee ffdZdedeeee ffdZdeee fdeee ffdZdZd eefd!Z d"ed#eefd$Z!d;d%ed&e"deefd'Z#d(ed)eedeeeffd*Z$d(ed+eee fdefd,Z%d(edeefd-Z&d(ed.eeefdeefd/Z'd%ed.eeefdefd0Z(d.eeefdeefd1Z)d2eee fd.eeefde*fd3Z+d4e"d.eeefdede*fd5Z,d2eee fd.eeefd(edefd6Z-deedeee*ffd7Z.d4e"deed8ede*fd9Z/y:)< VectorStorez0FAISS-based vector store with BM25 hybrid searchr;c||_g|_g|_d|_d|_g|_g|_d|_t|_ i|_ g|_ i|_ |j|j|_|j!|j"|_|j&dz |_|j&dz |_|j&dz |_|j/y)Nrzfaiss_index.indexzdocuments_metadata.jsonzembeddings.npy)r; documents embeddingsindexrsearchable_texts doc_lengthsavg_doc_lengthrword_doc_freqs term_df_cache doc_profilessource_profiles_load_source_ranking_configrsource_ranking_load_query_expansion_configrquery_expansionsr index_file metadata_fileembeddings_file _init_modelr3s rDrzVectorStore.__init__is   "%i!">>v?[?[\ $ A A&B_B_ `!**-@@#--0II%//2BB rF config_pathrcDtjtjt} |r|j rt |dd5}tj |}dddttrsdD]9}t|j|ts#||j||;t|jdtr|d|d<td||S#1swYxYw#t$r}td|Yd}~|Sd}~wwxYw) z6Load curated source ranking config with sane defaults.r=r>r?N)rrrz$[RAG] Loaded source ranking config: z,[RAG] Failed to load source ranking config: )r!loadsdumpsDEFAULT_SOURCE_RANKING_CONFIGexistsrJload isinstancerr@updatelistrr,rCr config_datarNloadedkeyr.s rDrz'VectorStore._load_source_ranking_configs jj,I!JK  F{113+sW=*!YYq\F*fd+>A%fjjot<',33F3K@A"&**^"r?N)rrrrrz%[RAG] Loaded query expansion config: z-[RAG] Failed to load query expansion config: )r!rrDEFAULT_QUERY_EXPANSION_CONFIGrrJrrrr@rrrr,rs rDrz(VectorStore._load_query_expansion_configs jj,J!KL  G{113+sW=*!YYq\F*fd+qA%fjjot<',33F3K@A"&**]";TB5;M5J M2A+OP** G A!E F F Grctr# td|_d|_t d|jd|_t dyy#t $r}t d|d|_Yd}~Cd}~wwxYw)zInitialize embedding modelzall-MiniLM-L6-v2iz&[RAG] Loaded SentenceTransformer modelz"[RAG] SentenceTransformer failed: NzT[RAG] Using hash-based embeddings (install sentence-transformers for better results))HAS_SENTENCE_TRANSFORMERSrr embedding_dimrr,)rCr.s rDrzVectorStore._init_modelsu $ "01CD %(">@ :: !$D  h i   ":1#>?!  "s"A A2A--A2r+c|jr|jj|dStj|jj }t j |t jjt j}|t jj|dzz S)zGet embedding for textT)normalize_embeddings)dtypeg& .>) rencodehashlibsha384digestnp frombufferuint8astypefloat32linalgnorm)rCr+harrs rD_get_embeddingzVectorStore._get_embeddings ::::$$T$E Et{{}-446A--299"**EC"))..-45 5rFtextsc|jr|jj|ddStj|Dcgc]}|j |c}Scc}w)z!Get embeddings for multiple textsT)rshow_progress_bar)rrrarrayr)rCrts rD_get_embeddings_batchz!VectorStore._get_embeddings_batchsN ::::$$UY]$^ ^88UCT003CD DCsAc|jjsy t|jdd5}tj||_dddt rH|jjr.tjt|j|_ nG|jjr-tjt|j|_|jt!dt#|j dy#1swYxYw#t$$r}t!d |Yd}~yd}~wwxYw) z!Load existing index and documentsFr=r>r?Nz [RAG] Loaded documentsTz[RAG] Failed to load index: )rrrJr!rr HAS_FAISSrfaiss read_indexrPrrrr_prepare_searchable_textsrr*r,rCrNr.s rDrzVectorStore.loads!!((* d((#@ .A!%1 .T__335"--c$//.BC %%,,."$''#d.B.B*C"D  * * , M#dnn"5!6jA B . .  04 5 s/D,D CD, D)%D,, E 5EE cz t|jdd5}tj|j|dddddt r@|j 4tj|j t|jn^t|jdkDrFtjt|jtj |jt#d t|jd y#1swYxYw#t$$r}t#d |Yd}~yd}~wwxYw) zSave index and documentswr>r?Fr# ensure_asciiindentNrz [RAG] Saved rz[RAG] Failed to save index: )rJrr!dumprr rr  write_indexrPrr*rrsaverrrr,r s rDrzVectorStore.saves 6d((#@ KA $..!%J KTZZ3!!$**c$//.BCT__%)D001288DOO3LM LT^^!4 5Z@ A K K 6 04 5 5 6s.D$D CD DD D:"D55D:c Ng|_g|_t|_i|_g|_i|_g}i}|jD]}|j|}|j}|jj||jjt|j|jjttj d||j#||}|j||j%|||j'||_|Dcgc]}|j)|c}|_|jr1t+|jt|jz |_yd|_ycc}w)zBuild searchable text cacherrN)rrrrrrrr_build_searchable_textrBrVr*rsrrrrr_build_preliminary_profile_update_source_aggregate_finalize_source_profiles_build_doc_profilerkr)rC prelim_docssource_aggregatesrXr+searchable_textprelims rDr z%VectorStore._prepare_searchable_textssg "%i! >> EC..s3D"jjlO  ! ! ( ( 9    # #C(=(=(?$@ A    & &s2::j/+R'S T44S/JF   v &  ) )*;V D E $==>OPKVWT44V<WOSO_O_c$"2"23c$:J:J6KKefXs=F"rXc g}|jdd}|rXtjdd|tj}|j |j ddj dd|jdr|j |d|jd r)|j t |jd |jd rH |j dj|jd gDcgc] }t |c}|jd rH |j dj|jd gDcgc] }t |c}|j |jd ddj|Scc}w#t$rYwxYwcc}w#t$rYVwxYw) z#Build searchable text from documentrrz \.(pdf|txt)$rP_r-rsummary key_pointsthemesr) r@rrsubrrVreplacerPrHr,)rCrXpartsr clean_sourceprs rDrz"VectorStore._build_searchable_textsx2& 66/2vR]]SL LL--c37??SI J 779  LLY ( 779  LLSWWY/0 1 77<   SXXsww|R7P&Q!s1v&QRS 778   SXXswwx7L&M!s1v&MNO  SWWY+,xx'R  'N  sH *F7 F2  F79*G #G 5 G 2F77 GGG GGpatternsc,tfd|DS)z9Check whether text matches any regex-like source pattern.c3hK|])}tj|tj+ywr)rrrr)r9rr+s rDr;z3VectorStore._matches_any_pattern..+s"Sw299WdBMM:Ss/2)any)rCr+r+s ` rD_matches_any_patternz VectorStore._matches_any_pattern)sS(SSSrFrc | t|jdd}|j}t|jddj t|jddj}tfdtD}tfdt D}tfdt D}t fd tD} |||||| |j|t|j|t|j|t|j|td S) z7Build document-level signals before source aggregation.runknownrrrc3,K|] }|vsd ywr\rQr9termrs rDr;z9VectorStore._build_preliminary_profile..4sYAX1Yr`c3,K|] }|vsd ywr\rQr3s rDr;z9VectorStore._build_preliminary_profile..5s[$4?CZA`c3,K|] }|vsd ywr\rQr3s rDr;z9VectorStore._build_preliminary_profile..6sSd4?;RSr`c3lK|]+}|jxs|jdd-yw)NrSr)r9rr section_lowers rDr;z9VectorStore._build_preliminary_profile..8s:  NN= ) SW^^OET%>|Mf%g%)%>%>|Mf%g  rF aggregatesrc 0|d}||vr$dtdddd|d|d|d|dd ||<||}|dxxd z cc<|d |d xxd z cc<|d xx|d z cc<|dxx|dz cc<|dxx|dz cc<|dxx|drd ndz cc<y)z@Accumulate source-level stats used to tag corpus quality/topics.rrr>r?r@rA) doc_count doc_typesbetting_hits_totalbaseball_hits_totalprop_hits_totalfront_matter_countr>r?r@rArMrTrNrrOr:rPr;rQr<rRr=Nr)rCrKrraggs rDrz$VectorStore._update_source_aggregateJs!  #$Y&''(#$&''-.C'D(./E(F)/0G)H)/0G)H "Jv   KA K +,1, !VN%;;! !"f_&==" &"55 !&1B*CQJ!rFc i}|jD]\}}t|dd}|dr|djdddnd}|d|z }|d|z }|d|z } |d |z } |d } |d xs|d k\xs |d k(xr|dk\} |dxs|dk\} | dk\}|dxs|dvxr| }| xr | xs|dk\}|||| | | | | |||d ||<|j||||S)z7Turn source-level stats into stable quality/topic tags.rMrTrNrrrOrPrQrRr@r>g@rYg @r?g@rrA>rWrV) dominant_doc_typeavg_betting_hitsavg_baseball_hits avg_prop_hitsfront_matter_ratiois_data_dump_sourceis_betting_sourceis_baseball_sourceis_baseball_betting_sourceis_reference_sourceprop_focus_source)itemsmax most_common_apply_curated_source_rule)rCrKprofilesrrSrMrUrVrWrXrYrZr[r\r_r^r]s rDrz%VectorStore._finalize_source_profilescs%++-' FKFCC ,a0IIL[IYK 0 < M!$%9!:Y!F "%&="> )*O#s*O%1M6F#6M  *+,$+ !. 4 +,B$(AA#((' *;)o@R@nVgknVn &&7$4%6!.&8':%6&8.H':%6 HV   + +FHV4D EO' FRrFrprofilec>|j|}|s g|d<d|d<yt|jdg}||d<t|jdxsd|d<d|vrd|d <d |vrd|d <d |vrd|d <d|vrd|d<d|vrd|d<|d r |d rd|d<yyy)zAApply curated source tags/tiers from config on top of heuristics. curated_tagsr curated_tierNtagstierrTr[rr\rr_rr^rrZr])_find_curated_source_rulerr@rP)rCrreruleris rDrcz&VectorStore._apply_curated_source_rules--f5&(GN #&/GN # DHHVR()"&"%dhhv&6&C)"D  +/G' (  ,0G( ) d?+/G' ( $ -1G) * $ -1G) * & 'G4H,I48G0 1-J 'rFcd|jjdgD]x}t|jddj}t|jdd}|sJ|dk(r ||k(r|cS|dk(s^ t j ||r|cSzy#tj $rYwxYw)z5Find the first curated source rule matching a source.r match_typeexactrrregexN)rr@rPrBrrrerror)rCrrlrn match_values rDrkz%VectorStore._find_curated_source_rules''++NB? DTXXlG<=CCEJdhhw34KW$;)> W$yyf5# 6 xxs<BB/.B/c |jj|di}|d|d|d|d|ddk(xs|ddk\|dxs|dd k\|d |d S) z9Build lightweight relevance features used at search time.rrr:r;r<rYr%r?r#r=)rr:r;r< is_sports_docis_baseball_docr=source_profile)rr@)rCrrvs rDrzVectorStore._build_doc_profiles--11&2BBGz*">2#O4 ,#J/8;Zvn?UYZ?Z%&<=]AX\]A]%&78,  rFc~ |jj|jjdz}t|dd5}t j |j |dddddtj||jy#1swY*xYw#t$r}td |Yd}~yd}~wwxYw) zOPersist documents metadata safely without rewriting the FAISS index/embeddings..tmprr>r?Fr#rNz,[RAG] Failed to save metadata (checkpoint): ) r with_suffixrArJr!rrr>r'r,r)rCtmp_pathrNr.s rDsave_metadata_onlyzVectorStore.save_metadata_onlys F))55d6H6H6O6ORX6XYHhg6 K! $..!%J K JJx!3!3 4 K K F @D E E Fs0AB$B&(BBB B<$B77B<rc|sy|Dcgc]}|d }}|j|}tr|j$tj|j |_t j|jd}tj||jj|njt|jt jr?t|jdkDr't j|j|g|_ n||_ |j j#||j%t'dt|dt|j ycc}w)zAdd documents to the storeNrrrz [RAG] Added z documents. Total: )rr rr  IndexFlatIPrrrr normalize_L2addrrndarrayr*vstackrrr r)rCrccontentsnew_embeddings embeddings_nps rD add_documentszVectorStore.add_documentss *01QAiL1133H= zz!"..t/A/AB HH^4;;IFM   } - JJNN= )$//2::63t;ORS;S"$))T__n,M"N"0 f% &&( S[M)  ? # **4+=+=>DJ2;rt |j<d kDrt!j@|j<| }t+t-||j.j0z|j.j2t |j} t!jB|| dddd }|Dcgc] }|||f }}ng}g}g}|D]\}}|t |jk\r |j|}|t |jDkr|jD|ni}|t |jFkr|jF|nd }d }g}|D]7} | d j5|s|j| d|| drdndz }9t+d|}|jI||| }|jK|||}|jM|jOdi||}|j.jPtS|z|j.jT|zz|j.jV|zz} | |z } |j|d|jYD!cic]\}}!|dk7s ||!c}!}| tS|||||d|s|j|j[|j]dd|j]ddg}"i}#|D]z}$|$djOdd}%|#jO|%d |j.j^ksB|"j|$|#jO|%d dz|#|%<t |"k\sznt |"kr|D]{}$|$djOdd}%|#jO|%d |j.j^ksB|"j|$|#jO|%d dz|#|%<t |"|k\sz|"S|"Scc} wcc}wcc}!}w)zHybrid vector + keyword searchrr#z\b)r4r is_sportsr4TNrrrrrr'r?rvr)rmetadatascore vector_score keyword_score bm25_score intent_boost matched_termsc |dSNrrQxs rDz$VectorStore.search..xs ajrFrreversec |dSrrQrs rDrz$VectorStore.search..ys 7rFrrr1rT)0rrBrrr_build_query_profile_expand_query_for_retrievalr*rVcompileescaper SPORTS_TERMS _compute_idfrr rrrrr r~rrar;r1r/rrziprrrdotargsortrr _compute_bm25_compute_intent_boost_should_suppress_candidater@r(rOr)r*r`popsortr+)&rCrr query_lower query_words query_profileexpanded_querykeyword_patternswordr4r* idf_tablequery_embedding oversamplescoresindicesvector_results similarities top_indicesr candidatessuppressed_candidatesridxrXrerrrrrsuppress_candidatecombinedvr source_countsrrs& rDrzVectorStore.search s~~Ikkm jj[9 11+{K 99%O D4y1} '' !zz%"))D/*AE*I2==Y!%!5) ! D{"tGW3X!AfI3X'X '' !zz%"))D/*AE*I2==Y!%) %%&67 --n= / hh'89@@KO    /A <<_>_`DNN#J#jj//LOFG!#fQi"<=N$//2::63t;ORS;S!vvdooG  DKK@@@$++BcBcd' !jj6 {|DTrTJ @K!L1<?A"6!L!L!# "!/. ? L#c$..))..%C03c$:K:K6L0Ld'',RTG.1iikLdaQ)^QTL! %l 3!.( ,!.  "%,,Z^^-=>]. ?b 0$?""';T"J  Az]&&x;F  +dkk.O.OOq!(5(9(9&!(Dq(H f%w<1$   w"MXMs Y Y 9 YYrrct|}tfdtDtfdtDtfdtDt |dkdS)zCClassify the query so ranking can prefer the right document family.c3&K|]}|v ywrrQr9r4rs rDr;z3VectorStore._build_query_profile..s V!4 Vc3&K|]}|v ywrrQrs rDr;z3VectorStore._build_query_profile..s!X$$+"5!Xrc3&K|]}|v ywrrQrs rDr;z3VectorStore._build_query_profile..sQtt{2Qrr%) wants_bettingwants_baseball wants_props is_generic)rr.rBrCrDr*)rCrr query_termss ` rDrz VectorStore._build_query_profilesQ+&  VAU VV!!XBW!XXQ?PQQk*a/   rFrlct|jddj}t|jddj}|sy|dk(r||jk(S|dk(r|j|vS|dk(r( t j ||tj duSy#tj$rYywxYw) z.Check whether a configured query rule matches.rncontainsrrFrorpN)rPr@rBr rrrrrq)rCrrlrnrrs rD_query_rule_matcheszVectorStore._query_rule_matchess, ;<BBD $((7B/0668   +"3"3"55 5  #$$&+5 5   yyk2==IQUUU88  s &B33C C c2g}|jjdijD]g\}}t|ts|D]L}t |j j}|s-||vs2|jt |gi|S)z&Detect sports from configured aliases.r) rr@r`rrrPr rBrV)rCrrYsportaliasesalias alias_texts rD_detect_query_sportsz VectorStore._detect_query_sportss"3377LRRT NE7gt,   Z--/557 * ";MM#e*-    rFrcg}|jjdi}|jdr!|j|jdg|jdr!|j|jdg|jdr!|j|jdg|j|}|jjdi}|D]#}|j|j|g%|jjd gD]G}t |t s|j ||s'|j|jd gIg} t} |D]E} t| j} | s| | vs$| j| | j| G| S) zLResolve named expansion groups from intent, sport, and explicit query rules.rrrrrrrrr add_groups) rr@rrrrrrrPr rVr) rCrrrintent_defaultsdetected_sportsrrrldedupedseenr group_names rD_resolve_expansion_groupsz%VectorStore._resolve_expansion_groupss//334KRP   _ - MM/--i< =   ] + MM/--gr: ;   - . MM/--j"= >33K@#44889OQST$ ?E MM.225"= > ?))--mR@ :D$%$*B*B;PT*U dhh|R89 :u %EU))+Jj4z*$  % rFc|j}|j||}|jjdi}g}|D]Y}|j|g}t |t s&|D]/} t | j} | s|j| 1[|s|Sg} t} |D])} | | vs| j| | j| +|ddj| S)zGAdd light retrieval hints so short queries pull the right corpus slice.rr) rBrrr@rrrPr rVrrrH)rCrrrrr expansionsrvaluesvaluer+rritems rDrz'VectorStore._expand_query_for_retrievalskkm // ]K,,00D   ,E!%%eR0Ffd+ ,5z'')%%d+ ,  ,Lu D4t$  #((7+,--rFcg}|jdr|jd|jdr|jd|jdr|jd|jdrD|jdr3|jdr"|jdr|jd|S) zCMap query intent to weight keys used by the curated ranking config.rrrrrrrr)r@rV)rCrkeyss rD_query_weight_keyszVectorStore._query_weight_keyss   _ - KK "   - . KK #   ] + KK   l +!!/2!!"23!!-0 KK+ , rFrvc d}|j|}t|jdxsd}|jjdi}|jjdi}|D]u}|t |j|ij|dz }|jdgD]0} |t |j| ij|dz }2w|S)z2Apply curated source tier/tag weights from config.rrhrrrrg)rrPr@rrO) rCrvrbonus weight_keysrjrrrtags rD_curated_source_bonusz!VectorStore._curated_source_bonuss--m< >%%n5BC**..~rB ))--mR@  GC U<++D"599#sCD DE%))."= G{sB7;;CEFF G G  rFrcn|t|jk\ry|j|}|jdi}|j||}|jdrT|jdr|dz }n |dr|dz }|t |dd d zz }|jd r |dd k(r|d z}|jdr6|jdr|dz }n |dr|dz }|t |dd d zz }|jdrS|t |dddzz }|jdr|dz }|jds |ds|dz}|dd k(r|d z}|jd r8|jdr'|jdr|jdr|dz }|jd r_|jdrN|jdr=|jdr|dz }|jdr|jds|dz}|jdr)||jds|jdrdndz}|d r|dz}|jd!ddkDr|d z}|j d"d#k\s|j d$d%k\r|dz}t d&t d'|S)(zPBoost or penalize candidates based on query intent and obvious low-value chunks.rrvrr[gQ?rtrr:rygQ?rrrrr\g{Gz?rur;rr<r%g{Gz?r_rrr]gQ?r^rZgffffff?rr=rY copyrightr#ztable of contentsrTgr')r*rr@rrcountra)rCrrrrervboosts rDrz!VectorStore._compute_intent_boosts #d''( (##C( %5r:**>=I   _ -!!"56 )  S0!4t; ;E  .7>3Ja3O    - .!!"67 *+  S115< .esQDDK/QrT)r@r.HISTORY_INTENT_TERMS)rCrvrrrrr wants_historys ` rDrz&VectorStore._should_suppress_candidateXss!!"78#'' 6 "&&|4 %))/: Q.s#Zt) @S@STX@YZs$$r'rr) r*rrrr fullmatchrr@rkrmathlogra)rCr+r total_docsr4dfidfr*s @rDrzVectorStore._compute_idfrs ( ?  ,AV9Dy t)))''-fd+((,,T15+-""4(Zt'<'<ZZ+-""4(((JOc1b3h?#EFC!#smIdO ,"rFrc|r|t|jk\ry|j|}|t|jkr|j|nt|j}|jxs|xsd}d}|D]q}t|dj |} | dk(r&|j |dd} | tdzz} | tdtz t||z zzzz} || | | z zz }s|S)z!Compute BM25 score for a documentrrTrrr4r) r*rrrsrrr@BM25_K1BM25_B) rCrr+rdoc_textdoc_lenavg_lenbm25r*tfr numerator denominators rDrzVectorStore._compute_bm25s3#d&;&;"<<((-+.T5E5E1F+F$""3'CPXP^P^P`La%%55A 4AQy\))(34BQw--& 3/Cgm,Iw#,7WCT9U*UVVK C9{23 3D 4 rFN)r,)0rGrHrIrJrrrr rPr rrrrrrr rrrrr rr/rrrrcrrkrr{rrrNrrrrrrrrOrrrrrrQrFrDrrfs:y0tS#X(c3h( j6362::6E49EEd* 6g6$3BTTS TdT d S TRUWZRZ^ :K4T#s(^8K3LKVZ[^`c[cVdK2,Dd38n9L4M,RVWZ\`adfiai\jWjRk,\99tCH~92c3h8P"  c3h  DcN  FTDJT4'#'4:'r?r@r3r5r3s rDrzOCRService.__init__s; " /AC^ _!#0CE`!arFr4rctsy|j|j|d}|r|S|j|j|d}|S)zCTry OCR extraction with DeepSeek (primary) and Tesseract (fallback)NDeepSeek Tesseract)r&_try_ocrr3r5)rCr4r+s rDrzOCRService.extract_textsI}}T22IzJ K}}T33Y L rFurl service_namec < td|d|dt|d5}d|j|dfi}tj||d}d d d j d k(r|j }|jd r4|jd r#td |dt|d d|d S|jd r#td |dt|d d|d Sy #1swYxYw#t$r}td |d|Yd }~y d }~wwxYw)zTry OCR servicez [RAG] Trying z OCR at ...rbfilezapplication/pdfiX)filesr"Nr#successr+z[RAG] z OCR extracted z charsz OCR failed: ) rrJrr'r(r)r!r@r*r,) rCrr4rrNrr-r8r.s rDrzOCRService._try_ocrs2 ; M,xuC@ Ai& H!)..!5F!GH#==E3G H##s*}}88I&488F+;F<.DL@Q?RRXYZ<'XXf%F<.DL@Q?RRXYZ<' H H ; F<. aS9 : : ;s5C7*C+ A+C763C7+C40C77 DDDN) rGrHrIrJrrrrrPrrrQrFrDr r sP`byb d x} CDQT rFr ceZdZdZdefdZdedefdZdededefdZ dd ed e de efd Z ded ede efd Zd ede efdZdededefdZdde ede efdZy)EnrichmentServicez&Service for LLM-based chunk enrichmentr;c||_t|_ttj j dd|_d|_y)NENRICHMENT_BUDGETi?Br) r;rllmrNr>r?r@enrichment_budgetenrichment_usedr3s rDrzEnrichmentService.__init__s7 <!$RZZ^^4G%P!Q rFrrc|jdd}|rt|dkr|S|j||}|j|jkr:|j ||}|r&|j ||xjdz c_|j}|jdd|d<|jdg|d<|jdg|d<d|d <|S) z1Enrich a single chunk with LLM-generated metadatarr2rTr#r$r%Tenriched)r@r*_generate_basic_cardr r_enhance_with_llmrcopy)rCrrcardenhancedenriched_chunks rD enrich_chunkzEnrichmentService.enrich_chunks))Ir*#g,+L((%8   $"8"8 8--gt%alsobeeneachevenfromhaveintojustlikemoremostonlysomethanthatthemtheythisverywerewhatwhenwillwithyouraboutbeingcouldothertheirtherewherewhichwouldshouldbecausethroughryrrTc |dS)NrTrQrs rDrz3EnrichmentService._extract_themes..+s 1rFTrNr#) rrrrBr*r@sortedr`rrV) rCr+r.r stop_words word_countsr sorted_wordsrr%r4s rDr-z!EnrichmentService._extract_themess . =I   AD:%#d)q.$/OOD!$ Key Points: - - - Themes: theme1, theme2, theme3 Begin the card now:r#rrrz[RAG] LLM enhancement error: )rr_parse_llm_responser,r)rCrrZrr-r.s rDr%z#EnrichmentService._enhance_with_llm7s # 9  $ 7xx((CS(QH//99  7 1!5 6 6 7s0A A% A  A%c |syi}tjd|tj}|r%|jdj dd|d<g}tjd|tj}|r|jdj j d}|D]\}|j }|j dr|ddj }|s:t|d kDsI|j|dd ^|r|dd |d <tjd |}|rT|jdj d D cgc]} | j } } | D cgc]} | s|  c} dd |d<|r|SdScc} wcc} w)z%Parse LLM output into structured cardNz)Summary:\s*(.+?)(?=Key Points:|Themes:|$)rTrr#z Key Points:\s*(.*?)(?=Themes:|$)r7r"rr,r$zThemes:\s*(.+),r%) rrrDOTALLrr rs startswithr*rV) rCr+result summary_matchr$kp_matchrrg themes_matchrr%s rDr]z%EnrichmentService._parse_llm_responseTs| "NPTVXV_V_`  - 3 3A 6 < < >t DF9  99@$ R NN1%++-33D9E 2zz|??3'8>>+DCIM%%d4Cj1  2 #-bq>F< yy!2D9 )5););A)>)D)DS)IJAaggiJFJ+17aQ7;F8 v)T)K7sF+F 3F originalr'cd|jddd|jdddd|jd d g}|jd r/|jd |d D]}|jd ||jdr&|jddj|d|jd|dj|S)z$Compose enriched content for storagez [Source: rUnknownz | Chunk r?]z Summary: r#rr$z Key Points:z- r%zThemes: z, z Original Content: r7)r@rVrH)rCrfr'rpoints rD_compose_enriched_contentz+EnrichmentService._compose_enriched_contentvs956iUX@Y?ZZ[ \B/0 1  88L ! LL 'l+ + r%\* + 88H  LL8DIId8n$=#>? @ ,XJ78yyrFNrc g}t|}t|D]c\}}|j|}|j||r ||dz||dzdzdk(sBt d|dzd|d|j de|S)z(Enrich all chunks with progress trackingrTr,rz[RAG] Enriched /z chunks (LLM used: ))r*r}r*rVrr )rCrprogress_callbackr#rhrrr)s rDenrich_all_chunksz#EnrichmentService.enrich_all_chunkssF !&) cHAu!..u5N OON + !!a%/A|q Awaw6I$J^J^I__`ab crF)rr)rGrHrIrJrrr r*rPr$rNr r-rr%r]rlrqrQrFrDrrs0!y! $4< C   . #C #S #c #D(4.: * * *D # T c &T tTXzrFri@MAX_CONTENT_LENGTHidlerENRICHMENT_CHECKPOINT_EVERYr,) statusprogressr%llm_used updated_at started_atr total_target done_targetcheckpoint_everyzenrichment_status.jsoncN tjtjdz}t|dd5}t j t |dddddtj|ty#1swY$xYw#t$r}td |Yd}~yd}~wwxYw) z4Persist enrichment status for resume after restarts.rxrr>r?Fr#rNz+[RAG] Failed to persist enrichment status: ) ENRICHMENT_STATUS_FILEryrArJr!renrichment_statusr>r'r,r)rzrNr.s rD_save_enrichment_status_to_diskrsA)556L6S6SV\6\] (C' 2 Ja II'q I J 834 J J A ;A3?@@As.4BA7"B7B<B B$ BB$c P tjrUttdd5}tj|}dddt t rtj|tjddtjdd tjd d tjd d tjd dtjddtjddtjdd tjdd tjdttjjddtjddk(rodtd<tjd xsdtd <t!j"t$j&j)dztd <t+yy#1swYxYw#t$r}td|Yd}~d}~wwxYw)zRRestore enrichment status on startup (and mark stale running jobs as interrupted).r=r>r?Nz([RAG] Failed to load enrichment status: rursrvrr%rrwrxryrrzr{r|rtr,running interrupted4Previous enrichment was interrupted. You can resume.Z)r~rrJr!rrrrrr,r setdefaultrNr>r?r@rnowrutc isoformatr)rNr8r.s rD!_load_enrichment_status_from_diskrs> ! ( ( *,cGD $yy| $$%!((.   62  Q/  B/  Q/  t4  t4  40  3  2  !3SHegi9j5klX&)3&3(#  ! !) , FE )$+3,,x||*D*N*N*PSV*V,'')4) $ $ > 8<==>s-&HG6-H6H;H H% H  H% FLASK_API_KEYzeventheodds-flask-api-key-2025c.tfd}|S)Nctjjdd}|tk7rt ddidfS|i|S)Nz X-API-KeyrrqzInvalid API keyi)rr r@rr)argskwargsapi_keyrNs rD decoratedz"require_api_key..decoratedsG//%%k26 m #G%678#= =$!&!!rFr)rNrs` rDrequire_api_keyrs! 1X"" rFz9[RAG] Initializing Advanced Sports Betting RAG Service...job_idrurvr%c ~t5|||tjd|t|<dddy#1swYyxYw)zUpdate upload progress)rurvr%updatedN) progress_locktimeupload_progress)rrurvr%extras rDupdate_progressrsB   yy{ #  #    s#3<r4rYc  t|dddtj|}|rt|j dkr#t|dddt j|}|rt|j dkrt|ddd y t|dd d t|d tj ||}tj||}|st|dddy t|D]"\}}||d<||d<t||d<||d<$t|dddt|dtj||t|dddtjt|dddt|dt||ttjy #t$rF}td|dd l} | j!t|dddt#|Yd }~y d }~wwxYw)z#Process uploaded file in background processingr,z Extracting text from document...rz%Text extraction failed, trying OCR...failedrz$Failed to extract text from documentN(z Extracted z characters. Chunking...z'No valid chunks extracted from documentrrrr<zCreated z! chunks. Generating embeddings...ZSaving index... completedzSuccessfully processed! Added z chunks.) chunks_addedrtotal_documentsz[RAG] Error processing file: Error: )r pdf_processorrr*r  ocr_servicerprr} vector_storerrrr,r traceback print_excrP) rr4rYr+rrrrr.rs rDprocess_file_backgroundr s6A b2TU)))4s4::<(3. FL"6] ^++I6Ds4::<(3. FHa1W X  bJs4ykIa2bc!55dHE))$9 FHa1Z [ "&) .HAu&E(O !E* $'KE. !%-E/ "  .  bHS[MIj2kl ##Hf5 b2CD K,S[M BV" 6 67   A -aS12!ws1vh-?@@ As&BF.AF.+CF.. G=7 Sports Betting RAG - Document Upload

🏆 Sports Betting RAG

Advanced Document Processing with FAISS + BM25 + DeepSeek LLM Enrichment

📊 Status
📤 Upload
📄 View Chunks
✨ Enrichment

📊 System Status

-
Total Chunks
-
Enriched
-
Raw
-
Files
-
Search
rQrQrFrDrrJsY rFz/healthGET)methodscjtdttjtt t dS)z Health checkhealthy)rur has_faisshas_sentence_transformers has_pymupdf)rr*rrr rrCrQrFrDhealthr) s0 |556%>"  rFz/statusc tdtjD}tt tjt |t t t rdndt rddSddS)zGet system statusc3@K|]}|jddyw)rrNr@r9rs rDr;zstatus..9 sF!!%%"%Fsrz hash-basedzFAISS + BM25 HybridzCosine + BM25 Hybrid)r total_filesrrembedding_model search_type)rrrrr*r r)sourcess rDruru5 slF|/E/EFFG |5567|%>4M0S_09,   @V  rFz/uploadPOSTcdtjvrtddidfStjd}|jstddidfSt |jj j }|dvrtddidfSt|j}tj|z }|jt|ttjdd }t|d d d tj t"|||fd }|j%td |d|ddfS)zUpload and process a documentrrqNo file providedrEmpty filenamer6r<z$Only PDF and TXT files are supportedNstartingrz%File uploaded, starting processing...T)targetrdaemonz"File uploaded. Processing started.rrr%rY)rrrrYrrArBrr;rrrPuuiduuid4r threadingThreadrstart)rext safe_filenamer4rthreads rD upload_filerE s8W]]"!345s:: == D ==!123S88 t}}  $ $ * * ,C ""!GHI3NN$DMM2M.IIIc)n r "FFJ+RS   &i /F  LLN 7!     rFz/progress/ct5|tvrtt|cdddS dddtdddddfS#1swYxYw)zGet upload progressNr1rz Job not found)rurvr%)rrr)rs rD get_progressrm sX 4 _ $?62344 $4 iQ?S TVY YY44s AAz/askc ztjdxsi}|jddj}|jdd}|st ddidfS|jd d }t j || }|s't d gtt jd dSg}d }t}g}|D]} | jdi} | jdd} |r| r| jdr|dz }d| jdd} | jdg} | r.|j| dd| ddj| ddz } | jdg}|r|j||j| |j| ddddj|}d} tjj!tjj!tjj#t$}tjj|dd }tjj'|rNt)|d!d"#5}|j+j}t-d$t|d%dddnt-d&|d(|d)|dd*d+|d,} t1}|j3|d-d./}|r|j}nd0|dd1z}g}|D]} | jdi} |j| jd4d5t5| d6d| ddd7| jdd| jdr| jdddd8nd| jdgd9t ||tt jt||t7|dd:|dd d;S#1swY1xYw#t.$r}t-d'|Yd}~Qd}~wwxYw#t.$r!}t-d2|d3|dd1z}Yd}~5d}~wwxYw)r?z[RAG] Injected z chars of live NFL contextz [RAG] Context file not found at z[RAG] Context injection error: zYou are the AI Guru, an expert sports betting assistant. Answer the user's question using the provided Context and Live Data. === LIVE REAL-TIME DATA (2025-2026 NFL SEASON) === z1 === RETRIEVED KNOWLEDGE (STRATEGY & BOOKS) === iz === QUESTION === a INSTRUCTIONS: 1. Prioritize 'LIVE REAL-TIME DATA' for specific 2025-26 standings, records, and current stats. 2. Use 'RETRIEVED KNOWLEDGE' for betting theory, concepts, and general strategy. 3. If the question asks about a specific team's CURRENT record (e.g. Patriots), use the Live Data. 4. Synthesize everything into a natural, helpful response. 5. If you cannot answer, say so, but provide relevant context. ANSWER:rr'r\z_I could not generate a specific answer (Grok API return empty). Here is the relevant context: i z[RAG] LLM generation error: z/Error generating answer. Here is the context: rr1rr#)rrpreviewr#r#r%r,)rrrchunks_returnedrr%r$)rget_jsonr@r rrrr*rrrrHrrVr>pathdirnameabspathrKrrJrKrr,rrroundr)r8rrrr context_partsr all_themesall_key_pointsr=meta is_enrichedpartr$r%context live_contextr context_pathrNr.r llm_servicegenerated_answerrrs rDaskrv sn   4 ( .BDxx B'--/H88ND1L !7893>> aA!!(a!0G P"<#9#9:    MNJN 5uuZ$hhz51 KDHHY,? a N"488Ir#:";>, 'lC': Wa vvx~~/ L(9'::TUV W W5l^D E  $  F, V l &//4UX/Y %++-Fx|CDIEI|JJF G   uuZ$hhx31W:q)|DS)U38<8Ktxx 2.t4QUhhx,      |556w<(z"3B'$Ra( o W W 5 /s3445B V ,QC01DwuPT~UVsI>B+O+)7O O+9PO(#O++ P 4PP  P:P55P:z/searchc6tjdxsi}|jddj}|jdd}|st ddid fSt j || }t ||tt jd S) zSearch documentsTrrrrr,rqzQuery is requiredrr)rrr) rrr@r rrrr*r)r8rrrs rDrr s   4 ( .BD HHWb ! ' ' )E bA !456;;!!%1!-G |556 rFz /documentsc i}tjD]c}|jdd}||vrdd|jddd||<||dxxdz cc<|jdsT||d xxdz cc<ettj}t d tjD}t |j Dcgc] \}}d|i| c}}t|||t|dkDr||z d znddd Scc}}w) z)List all documents with enrichment statusrr1rr) chunk_countrrrrTr#rc3DK|]}|jdsdywr#rTNrrs rDr;z!list_documents.. sPqaeeJ>OP  r)r total_sourcesrtotal_enrichedenrichment_percentage)rrr@r*rkrr`r)rrXrrrrrs rDlist_documentsr sG%% 39-   "#!$)!DGFO  &!+& 77:  FO, - 2 - 3|--.LPL$:$:PPN 5<]]_ETQx(a(EW$(!&P\_`P`(E(Kfgij!k  Es D z/chunkscPtjjd}tjjddjdk(}tjjddjdk(}t tjjdd}t tjjdd }t j }|r$|Dcgc]}|jd|k(s|}}|r"|Dcgc]}|jds|}}n#|r!|Dcgc]}|jdr|}}t|}|dz |z}||z} ||| } g} t| | D]\} } | j| | jdd | jd | | jdd | jdr| jddddnd| jdgdd| jdgt| jdddkDr| jdddddzn| jddt| jddd t| |||||zdz |zd|||ddScc}wcc}wcc}w)z)List chunks with pagination and filteringrr#rtruerawr:rTper_pager)rr1rFr#Nr#r$r%r%ri,r) rrrr#r#r$r%content_previewcontent_length)r:rrhrI)r enriched_onlyraw_only)r paginationfilters) rrr@rBrNrrr*r}rVr)rrrr:rfilteredrrhrend paginatedrrrXs rD list_chunksr % s\\  h 'FLL$$Z4::<FM||r*002fAs$*JJJ'J4J# J#z/chunks/c |dks|ttjk\rtddidfStj|}t||j dd|j d|j d|j d d |j d |j d g|j d g|j ddt|j dd|j d|j dd S)z$Get a single chunk with full detailsrrqzChunk not foundrrr1rrr#Fr#r$r%rrrr) rrrrr#r#r$r%rrrr)r*rrrr@) chunk_indexrXs rD get_chunkr _ sQ+\-C-C)DD!234c99   -C ''(I.GGJ'1GGJ.779%gglB/''(B'779b)cggi45779%/   rFz$/chunks/by-source/ct|}tjDcgc]}|jd|k(s|}}|st ddidfSt d|D}t |t |||Dcgc]r}|jd|jdd|jd r|jd d d d nd |jd g|jdd d d dtc}dScc}wcc}w)z$Get all chunks for a specific sourcerrqzSource not foundrc3DK|]}|jdsdywrr)r9rs rDr;z'get_chunks_by_source.. s@qaeeJ.?@rrr#Fr#rNr#r%r)rr#r#r%r)rrenriched_chunksr)rrrr@rrkr*)r safe_sourcerrrrs rDget_chunks_by_sourcerw s"+.K%// RA155?k3Qa RF R !345s::@F@@N F )   j)j%056UU95EquuY+DS14eeHb) uuY3DS9     SsC6C66A7C;z/delete/DELETEcNt|}ttj}tjDcgc]}|j d|k7s|c}t_|ttjz }|dkDrtjrtjDcgc]}|d }}tj |}t rrtjtjt_ |jd}tj|tjj|n]|t_nQt r-tjtjt_ ntj gt_tj#tj%t&j(|z }|j+r|j-t/dd|d|d|d St/d d id fScc}wcc}w) zDelete a documentrrrrTzDeleted z (z chunks))rr%chunks_removedrqzDocument not foundr)rr*rrr@rr r r}rrrr~rrrrr rr;rrunlinkr) rYroriginal_countr removed_countrrrr4s rDdelete_documentr s$H-M//0N)5)?)?dA155?VcCcadL"S)?)?%@@Mq  ! !.:.D.DE) EHE%;;HEJ%*%6%6|7Q7Q%R " * 1 1) < ""=1""&&}5*4 '%*%6%6|7Q7Q%R "*,((2, '..0NN]2        !-=/J+   G12 3S 88Ie FsHH% H"z/reloadcttjtdttjdS)zReload the indexT)rr)rrrr*rrQrFrDreloadr s3 |556 rFz/enrichct5tjddk(r7t1tj rt dtddfcdddStjddk(rndtd<tjdxsd td<t jtjjd ztd <tdddtjd xsi}|jdfd}tj|d atj!t d ddddfS#1swYuxYw)z,Enrich documents with LLM-generated metadatarurNzEnrichment already running)rqruirr%rrrxTrrc 4 ttjjdtjddxsd} t j rAtt jDcgc]\}}|jdk(s|}}}n+tttt j}t|}|dk(rt5dtd<dtd <d td <tjtd <td<dtd <dtd<t!j"t$j&j)dztd<|td<t+dddyt-d|D}||z }t!j"t$j&j)dz}t5dtd<|rt||z dzndtd <d|d|d|dtd <tjtd <tjdxs|td<|td<td<|td <|td<|td<t+ddd|dkrt5dtd<dtd <dtd <tjtd <t!j"t$j&j)dztd<t+dddyd}|D]} t j| } | jdr)tj/| } | j1| t!j"t$j&j)dz| d< t j3| } tt j4tt jk(r!| j7t j4| <tt j8tt jk(rt j8| } t| j;}|t j8| <tt j8}|dkDr%t j<|z| z |z|z t _|dz }||z}|rt||z dznd}t5dtd<|td <d|d|d|d|d td <tjtd <t!j"t$j&j)dztd<|td<|td<ddd||zdk(sit j?t5t+dddt j?t5dtd<dtd <d|d |dtd <tjtd <t!j"t$j&j)dztd<|td<t+dddy#t$rY wxYwcc}}w#1swYyxYw#1swYxYw#1swYyxYw#t$rYwxYw#1swY:xYw#1swYxYw#1swYyxYw#t$r}tAd!|ddl!}|jEt5d"td<d#tG|td <tjtd <t!j"t$j&j)dztd<t+dddn#1swYnxYwYd}~yYd}~yd}~wwxYw)$Nrtr|r,rrrrurrvzNo chunks found to enrich.r%rwrzr{rrxc3fK|])}tj|jds&d+ywr)rrr@)r9rs rDr;z;enrich_documents..run_enrichment.. s*"j @V@VWX@Y@]@]^h@i1"js'11rzResuming enrichment: rnz' already enriched. Enriching remaining rryz0No chunks need enrichment (all already enriched)r# enriched_atrTz Enriched z chunks (this run: rozSuccessfully enriched z chunks (target: z[RAG] Enrichment error: rr)$rNr>r?r@rrr r,r}rrranger*enrichment_lockenrichment_servicer rrrrrrrkr*rrrrBrrsrr{rrrrP)r|rrtarget_indicesrzalready_enriched remainingryenriched_this_run doc_indexrX enriched_docnew_textold_lennew_lenndonervr.r source_filters rDrun_enrichmentz(enrich_documents..run_enrichment sz 2"2::>>2OQbQfQfgy{}Q~RECE$F G  668 09,:P:P0Q!v1UVUZUZ[cUdhuUu!!v!v!%eC 0F0F,G&H!I~.Lq $ 62=%h/47%j13O%i04F4V4V%j12?%h/89%n578%m46>ll8<<6P6Z6Z6\_b6b%l3ll8<<6P6Z6Z6\_b6b%l335 6 ! +) : ",,Y777:&1>>sC  <(%-\\(,,%?%I%I%Kc%QM" +BB3GH<889SAWAW=XXCK>>CS 55i@<334LE 00; 8 89q5;G;V;VYZ;Z]d;dgn;nrs:sL7"Q&!'*;;?K3| 3s:;QR$M2;%h/4<%j15>tfAl^Sfgxfyyz|E{FFG4H%i04F4V4V%j16>ll8<<6P6Z6Z6\_b6b%l37;%m4.process_and_enrich..enrich_progress s=3%2566 c=MgYVWX]W^^a;bcrFUzAdding to vector store..._rrz$Successfully processed and enriched z chunks!)rrwz [RAG] Error: r)rrrr*r rrprr}r"rqrrrr r,rrrrP) r+rrrrr=rr.rr4rrs rDprocess_and_enrichz,enrich_on_upload..process_and_enrich s6 E FL"6J K --i8D3tzz|,s2 b/J"// :3tzz|,s2!5MN FL"6L M$99$ NH"--dH=F!5PQ%f- 25"/h$%j!(+F n%)1o&  2 FL" 3v;-Ob6c d d1BB6?[O FL"6Q R  ' '  G FL"6G H      S6s?7K6LHU 1+;;   E M!% &     ! FHa73q6(1C D D  Es&BGA GC!G H sysr!rrrrrrrqueue collectionsrrrpathlibr functoolsr dataclassesr typingr r r rrflaskrrrrwerkzeug.utilsrrrrrFrGrHrIrrrBrCrDrrrrErrrRrC ImportErrorpdfminer.high_levelrrFrEPyPDF2rrGnumpyr HAS_NUMPYsentence_transformersrrr r r'r&rr0rr rrGappr;rrrr"rrNr?r@rLockrr!Queueprocessing_queuer1rr~rrrrrrrPrrrouterrrurrrrrr r rrrr3r5rArEr*rrunrQrFrDr\s   '!33>>*  ,8,8 ,8`!$sdhi|}~#sdhi|}~3$dexy z!SDfgz{|"cTgh{|}   "tdY]^"tZ^_$u5Q t$&d;)51!&US !&   .##$% ="B   BJJ/BJJ'7BJJ)2==9BJJ. >BJJ("--8BJJ,bmm<BJJ%r}}5   KDL J I &9 $ I L22jMMfz z @++bAAN Ho#4   V$ 6"  &v.BJJNN+H"MN    ).."5;;=)),DDA *F 0PQ AB !#  C      c  8AC8AD8AC8A~3[ [ |9ug&'9ug& ' 9vh'# (# L %1Z2Z6F8${%{x9vh'($<%)*89ug&6'6r &89. 1E7CD0 $x.@A*9B*9Z9vh'(9vh'Z (Z z %1*2*  fX.U /U v z rzz~~j$/ 0D Gv NO M)$<=VV?W  W WW