
    y]iȳ                     2   d Z ddlZddlZdej                  v xs, ej                  j                  dd      j                         dv ZddlZddl	Z	ddl
Z
ddlmZmZ ddlmZ 	 ddlZd	ZdZ ee      j.                  j.                  dz  dz  Z ee      j.                  j.                  dz  dz  Zej                  j                  dd      Zej                  j                  dd      ZdedefdZdddgdddddgdddddgddddgddd Zd!edefd"Z d9d#ed$e!d%e!d&e!de"f
d'Z#d:d#ed$e!d%e!d&e!def
d(Z$d$e!de%fd)Z&d#ed*e"defd+Z'd,e!d-e!d#edefd.Z(d;d/ed#ed$e!d0edef
d1Z)d2ede"fd3Z*d<d#ed4e"d5e+fd6Z,d7 Z-e.d8k(  r e-        yy# e$ r ddlZddlZd
ZY -w xY w)=a  
Fetch sports data from BallDontLie API and store it in the betting database format.

This script fetches game data for the 2024 season and converts it to the
format expected by sportsBettingData.ts for backtesting.

Usage:
    python scripts/fetch_balldontlie_data.py [sport]
    
    sport: nba, nfl, nhl, mlb, or all (default: all)

Environment:
    BALLDONTLIE_API_KEY: API key for BallDontLie (required)
    Nz--odds-backfill-allTHE_ODDS_BACKFILL_ALL )1trueyes)datetime	timedelta)PathTFzhttps://api.balldontlie.iodatabettingballdontlie_cacheBALLDONTLIE_API_KEYz$db60bdff-30db-4e3c-999a-9ce31d0a6c6cTHE_ODDS_API_KEYnamereturnc                     | sy| j                         j                         }dD ]  }|j                  |d      } dj                  |j	                               }ddddd}|j                  ||      S )	Nr   ).,'" zlos angeles ramszlos angeles chargersznew york giantsznew york jets)zla ramszla chargersz	ny giantszny jets)striplowerreplacejoinsplitget)r   nchaliasess       !scripts/fetch_balldontlie_data.py_norm_team_namer"   0   su    

A" IIb"A%-&"	G ;;q!    z/v1    znba_historical.json)
api_prefixseasonsoutput_filez/nfl/v1znfl_historical.jsonz/nhl/v1znhl_historical.jsonz/mlb/v1zmlb_historical.jsonnbanflnhlmlburlc                    dt         i}t        r8t        j                  | |d      }|j	                          |j                         S t        j                  j                  | |      }t        j                  j                  |d      5 }t        j                  |j                         j                               cddd       S # 1 sw Y   yxY w)z#Make an API request to BallDontLie.Authorization   )headerstimeout)r2   r3   N)API_KEYUSE_REQUESTSrequestsr   raise_for_statusjsonurllibrequestRequesturlopenloadsreaddecode)r.   r2   responsereqs       r!   make_requestrC   X   s    (G<<WbA!!#}}nn$$S'$:^^##C#4 	8::hmmo4467	8 	8 	8s   	1CCsportseasonper_page	max_pagesc           	      ,   t         j                  |       }|st        d|         g S g }d}d}t        d| j                          d| d       ||k  r|d| nd}t         |d	    d
| d| | d| 	}	t        d| ddd       	 t        |	      }
|
j                  dg       }|st        d       	 |S |j                  |       t        t        |       dt        |       d       |
j                  di       xs i }|j                  d      }|j                  d      }||}|dz  }n|t        |      }nt        d       	 |S t        j                  d       ||k  r|S # t        $ r%}t        d|        |dk(  rg cY d}~S Y d}~|S d}~ww xY w)zFetch ALL games for a given sport and season.

    BallDontLie APIs use different pagination mechanisms:
    - meta.next_page (page-based)
    - meta.next_cursor (cursor-based, e.g. NFL)
    Unknown sport:    N	Fetching  games for 
 season...z&cursor=r   r&   z/games?seasons[]=z
&per_page=&page=z  Page ...r   Tendflushr   done (no more data)z games (total: )metanext_cursor	next_pageu#     ✓ Completed - all pages fetched333333?error: )SPORT_CONFIGr   printupperAPI_BASE_URLrC   extendleninttimesleep	Exception)rD   rE   rF   rG   config	all_gamespagecursorcursor_partr.   r   gamesrU   rV   rW   es                   r!   fetch_all_gamesrk   f   s    e$Fw'(	IDF	Iekkm_Kxz
BC
)
-3-?)Rvl344EfXZX`Wabmanntuytz{vS!s$7	$DHHVR(E+,4 1 U#SZLI/?qAB88FB'-2D((=1K-I&$	&9~;<  JJsO; )
J   	GA3- qy		s+   *E% 5BE% E% %	F.FFFc                    t         j                  |       }|si S i }d}t        d| j                          d| d       ||k  ru| dk(  rt         |d    d| d| }n/| d	k(  rt         |d    d
| d| d| }nt         |d    d| d| }t        d| ddd       	 t        |      }|j                  dg       }	|	st        d       	 |S |	D ]q  }
|
j                  d      }|s||vr	dddd||<   |
j                  dd      j                         }|dk(  s|dk(  r	|
||   d<   W|dk(  r	|
||   d<   e|dv sj|
||   d<   s t        t        |	       dt        |       d       |j                  di       }|j                  d       }|t        d!       	 |S |dz  }t        j                  d"       ||k  ru|S # t        $ r}t        d#|        Y d}~|S d}~ww xY w)$zxFetch REAL odds from BallDontLie API for a given sport and season.
    Returns a dict mapping game_id -> odds data.
    rJ   rK   z REAL odds for rM   r*   r&   z/odds?per_page=rN   r+   /odds?season=z&week=1&per_page=z  Odds page rO   r   TrP   r   rS   game_idN)	moneylinespreadtotaltyper   2wayro   rp   )
over_underrq   totalsrq   z! entries (total games with odds: rT   rU   rW   u(     ✓ Completed - all odds pages fetchedrX   rY   )rZ   r   r[   r\   r]   rC   r   r_   ra   rb   rc   )rD   rE   rF   rG   rd   all_oddsrf   r.   r   	odds_list
odds_entryrn   	odds_typerU   rW   rj   s                   r!   fetch_real_oddsrz      sh    e$F	HD	Iekkm_OF8:
FG
)
E>!N6,#7"8zQWX\W]^Ce^!N6,#7"8fXM^_g^hhnosntuC "N6,#7"8zQWX\W]^CTF#&Ct<%	$D,I+,B O= ( @
$..3h.:>$Y],^) *vr : @ @ BI F*i;.F9C)+6"h.6@)(3"&GG5?)'2@ S^$$Ec(m_TUVW 88FB'D-I @B O AIDJJsO_ )
j O	  	GA3- O		s2   *F; F; %AF; 8AF; F; ;	GGGc                    t         j                  d      }|sg i fS i }t               }t        d|  d       t	        dd      D ]s  }t
         |d    d|  d| d	}	 t        |      }|j                  d
g       }|r|D ]  }|j                  d      }	|	s|j                  |	       |	|vs-|j                  d      |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      d
||	<    t        d| dt        |       dt        |D 
cg c]  }
|j                  |
      s|
 c}
       d       t        j                  d       t        r nv t        dt        |       d        g }t        d!t        |       d"       d#}|D ]  }		 t
         |d    d$|	 }t        |      }|j                  d
|      }|rC|j                  d%      r2|j                  |	i       }||d&<   d'|d(<   |j                  |       |dz  }t        j                  d)        t        d*| d       ||fS c c}
w # t        $ r)}dt        |      vrt        d| d|        Y d}~d}~ww xY w# t        $ r
}Y d}~d}~ww xY w)+zFetch NFL odds by iterating through weeks 1-22 (incl playoffs).
    Also fetches game details for each game with odds.
    Returns (games_with_odds_list, odds_by_game_id_dict).
    r+   zFetching NFL REAL odds for z season (weeks 1-22)...rJ      r&   rm   z&week=z&per_page=100r   rn   moneyline_home_oddsmoneyline_away_oddsspread_home_valuespread_home_oddsspread_away_valuespread_away_oddstotal_valuetotal_over_oddstotal_under_oddsvendor)
moneyline_homemoneyline_awayspread_homer   spread_awayr   rq   r   r   r   z  Week : z
 odds for  games皙?400z
: error - Nz  Found z unique games with oddsz  Fetching game details for z	 games...r   z/games/id
_real_oddsT_has_real_oddsg?z  Fetched details for )rZ   r   setr[   ranger]   rC   addr_   ra   rb   quota_exhaustedrc   strappend)rE   rd   rv   game_ids_with_oddsweekr.   r   rw   rx   rn   oidrj   games_with_oddsfetchedgame	game_oddss                   r!   fetch_nfl_odds_by_weekr      s   
 e$F2vH	'x/F
GHa #vl34M&PTvUbc 	$D,I"+ J(nnY7G*..w7"(22<..AV2W2<..AV2W/9~~>Q/R4>NNCU4V/9~~>Q/R4>NNCU4V)3)F3=>>BS3T4>NNCU4V*4..*B1HW-$ vRI'7z#VhF~sltlxlxy|l}sF~B  BA  AG  H  IJJsO ;#J 
HS+,--D
EF O	(-?)@(A
KLG% 	!N6,#7"8yIC$D88FD)D$LL"5	%.\")-%&&&t,1JJsO  
"7)6
23H$$G G  	CF"vZs34	2  		sP   8JJ+CJ1JJ)J4BK
J	KKK
	KKri   c                 
   t         st        d       i S ddddd}|j                  |       }|si S t        |D ch c]7  }|j                  d      s|j                  d      r'|j                  d      9 c}      }|si S t        d	| j	                          d
t        |       d       i }d}|D ]  }| d}	d| d}
dt          d|	 }	 t        rt        j                  |
|z   d      }|j                  dk7  rF|j                  dk(  r6	 |j                         }|j                  d      dk(  rt        d       d} |S 	 |j                         }n}t        j                  j                  |
|z         }t        j                  j                  |d      5 }t        j                  |j!                         j#                               }ddd       d}t%        t&              r|}nt%        |t(              rzt%        |j                  d      t&              r|j                  d      }nIt%        |j                  d      t(              r*t%        |d   j                  d      t&              r|d   d   }|s|D ]@  }|j                  d      }|j                  d      }|r|s+|j                  dg       xs g }|sD|d   }|j                  d g       xs g D ci c]  }|j                  d!      | }}|j                  d"      xs i j                  d#g       xs g }|j                  d$      xs i j                  d#g       xs g }|j                  d%      xs i j                  d#g       xs g }dx}}dx}}d&x} }!d}"|D ]N  }#|#j                  d'      |k(  r|#j                  d(      })|#j                  d'      |k(  s>|#j                  d(      }P |D ]r  }#|#j                  d'      |k(  r$|#j                  d)      }|#j                  d(d&      } ;|#j                  d'      |k(  sP|#j                  d)      }|#j                  d(d&      }!t ||t+        |       }||t+        |       }|D ]  }#|#j                  d)      }$|$|$}" n t-        ||||"g      s||||| |!|"d*d+||t/        |      t/        |      f<   C t1        j2                  d,        |S c c}w # t        $ r Y w xY w# 1 sw Y   "xY wc c}w # t        $ r Y Uw xY w)-zFetch odds from The Odds API historical endpoint and map them by (date, home, away).

    Uses a single snapshot per date (noon UTC) to keep usage reasonable.
    u4     ⚠️ THE_ODDS_API_KEY not set, skipping fallbackamericanfootball_nflbasketball_nbabaseball_mlbicehockey_nhl)r+   r*   r-   r,   datehasRealOddsz/  Fetching fallback odds from The Odds API for  (z
 dates)...Fz
T00:00:00Zz2https://api.the-odds-api.com/v4/historical/sports//odds?apiKey=zO&regions=us&markets=h2h,spreads,totals&oddsFormat=american&dateFormat=iso&date=r1   r4      i  
error_codeOUT_OF_USAGE_CREDITSuS     ❌ The Odds API quota exhausted (OUT_OF_USAGE_CREDITS). Stopping backfill early.TNr   	home_team	away_team
bookmakersr   marketskeyh2houtcomesspreadsru   r   pricepointthe-odds-apimoneylineHomemoneylineAway
spreadHome
spreadAwayspreadOddsHomespreadOddsAway	totalLinesourcer   )r   r[   r   sortedr\   r_   r6   r7   status_coder9   rc   r:   r;   r<   r=   r>   r?   r@   
isinstancelistdictfloatanyr"   ra   rb   )%rD   ri   	sport_map	api_sportgmissing_datesoutr   dsnapshot_isor.   paramsrespjpayloadrB   rA   eventsevhomeawayr   bmmr   r   r   ru   ml_homeml_awayr   r   spread_odds_homespread_odds_away
total_lineopts%                                        r!   fetch_the_odds_api_fallbackr   5  s?   
 DE	 &	I e$I	 5gaAEE&MRSRWRWXeRfAEE&MghM		;EKKM?"SQ^M_L``j
kl
CO fJ'B9+US,--|  ~J  }K  La	||C&L"=##s*''3.! $		A uu\26LL %&{ |26 %p Jw  M ))+nn,,S6\:^^++C+< C"jj)?)?)ABGC F'4( GT*gkk&148$[[0FF 3T:z'RX/J]J]^dJegk?l$V_V4F ;vvk*vvk*4  VVL"5;
!]57VVIr5J5PbRq155<?RR{{5)/R44ZDJ";;y17R<<ZLRPR!++h/52:::rJPb$((',00k6:: #3!
 1AuuV},"#%%.v$."#%%.	1 ! @AuuV},&'eeGn+,55$+?(v$.&'eeGn+,55$+?(@ &;+B#(#5"5K&;+B#(#5"5K AwB~%'
	 GWk:FG &-%,"-"-&6&6!+,	JQ-t/DEFe;z JJsOIfP Ja h2  ) ! !C C6 Sf  		s   SS"S>S8
1S>S8 AS82S&B)S81A#S8S3-C
S88A$S8AS8;AS8	S#S8"S##S8&S0	+S88	TTmargintotal_pointsc           	      l   |dk(  r| dkD  r%t        d| dz  z   d      }t        d| dz  z
  d      }n| d	kD  rd
| dz  z   }d| dz  z
  }np| dkD  rd| d	z  z   }d| dz  z
  }nZ| dk  r%t        d| dz  z   d      }t        d| dz  z
  d      }n0| dk  rd| dz  z   }d
| dz  z
  }n| dk  rd| dz  z   }d| d	z  z
  }nd}d}t        |  dz  dz        dz  }n|dk(  r| dkD  rd| dz  z   }d| dz  z
  }nr| dkD  rd| dz  z   }d| dz  z
  }n\| dkD  rd| dz  z   }d| dz  z
  }nF| d k  rd| dz  z   }d| dz  z
  }n0| d!k  rd| dz  z   }d| dz  z
  }n| dk  rd| dz  z   }d| dz  z
  }nd}d}| dkD  rd"nd#}n|d$k(  r| d%kD  rd
| d&z  z   }d| d'z  z
  }nr| d(kD  rd| d'z  z   }d| dz  z
  }n\| dkD  rd)| dz  z   }d*| dz  z
  }nF| d+k  rd| d'z  z   }d
| d&z  z
  }n0| d,k  rd| dz  z   }d| d'z  z
  }n| dk  rd*| dz  z   }d)| dz  z
  }nd}d}| dkD  rd"nd#}nl| dkD  rd
| dz  z   }d| dz  z
  }nF| dkD  rd| dz  z   }d| dz  z
  }n0| d-k  rd| dz  z   }d
| dz  z
  }n| dk  rd| dz  z   }d| dz  z
  }nd}d}t        |  dz  d(      }t        t        t        |d.      d/            t        t        t        |d.      d/            || t        |d      d0d1S )2z
    DEPRECATED: Estimate odds based on final score.
    WARNING: This creates information leakage - odds derived from outcomes are NOT valid for backtesting!
    Only use this for display purposes when real odds are unavailable.
    r+      i   ip      i^     i8         r   ij   iir      r-   i         iL      i~
   n   g      g      ?r,      r1      rJ   id   ii  ilive)r   r   r   r   r   r   )minmaxroundr`   )r   r   rD   r   r   r   s         r!   estimate_oddsr    s    ~B;$&1*-t4G#!,c2GaZfqj)GVaZ(GaZfqj)GVaZ(Gc\#!,c2G$&1*-t4Gb[VaZ(Gfqj)GaZVaZ(Gfqj)GGGVGaK!O,q0	%A:frk*GVb[)GaZfrk*GVb[)GaZfrk*GVaZ(Gb[Vb[)Gfrk*Gb[Vb[)Gfrk*GaZVaZ(Gfrk*GGG$qjdc	%A:frk*GVb[)GaZfrk*GVb[)GaZfrk*GVb[)Gb[Vb[)Gfrk*Gb[Vb[)Gfrk*GaZVb[)Gfrk*GGG$qjdc B;frk*GVaZ(GaZfqj)GVaZ(Gc\VaZ(Gfrk*GaZVaZ(Gfqj)GGGVGaK+ SWc!2D9:SWc!2D9:!"l<+ r#   r   	real_oddsc                    |dk(  r| j                  di       }| j                  di       }| j                  dd      xs d}| j                  dd      xs d}| j                  d      s| j                  d      r%| j                  d| j                  dd	            d
d nd	}|j                  d|j                  dd            }	|j                  d|j                  dd            }
n|dk(  r3| j                  di       }| j                  di       }| j                  di       }| j                  di       }|j                  dd      xs d}|j                  dd      xs d}| j                  d      r| j                  dd	      d
d nd	}|j                  d      xs8 |j                  d      xs% |j                  d      xs | j                  dd      }	|j                  d      xs8 |j                  d      xs% |j                  d      xs | j                  dd      }
n| j                  di       }| j                  di       }| j                  dd      xs d}| j                  dd      xs d}| j                  d      r| j                  dd	      d
d nd	}|j                  d      xs |j                  dd      }	|j                  d      xs |j                  dd      }
||z   }||z
  }|dk(  r|dk(  ry
||kD  rd}n
||kD  rd}nd}t        j                  | d| j                  dd	       d| j                               j	                         d
d }d}| j                  d       r| j                  d!      rx| j                  d!i       }d"}|j                  d#      }|j                  d$      }	 t        |j                  d%d      xs d      }	 t        |j                  d&d      xs d      }|}d
}d
}d
}d
}d'}d'}d
}|r|j                  d(      }|rN|j                  d)      xs |j                  d*      }|j                  d+      xs |j                  d,      }||d"}|j                  d-      }|rG|j                  d.      }|4t        |      }| }|j                  d)d'      }|j                  d+d'      }d"}|j                  d&      }|r1|j                  d/      xs |j                  d0      }|t        |      }|r@|rt        |      nd
|rt        |      nd
|||rt        |      nd'|rt        |      nd'|d1d2}nt        |||      } d3| d4<   | }|4|t        |      kD  r
|dk  rdnd}!n|t        |       k  r
|dk  rdnd}!nd5}!nd
}!|||kD  rd6}"n||k  rd7}"nd5}"nd
}"|| j                  d      ||||	|
||d
d
d
d
d
d
d
d
d8
||||!|"||d9d:S #  d}Y xY w#  d}Y xY w);a<  Convert a BallDontLie game to our betting format.
    
    Args:
        game: Game data from BallDontLie API
        sport: Sport code (nba, nfl, etc.)
        season: Season year
        real_odds: Dict with real odds data from BallDontLie odds endpoint
                   Keys: 'moneyline', 'spread', 'total'
    r,   r   r   
home_scorer   
away_score	game_dater   r   Nr   	full_namer   Unknownr-   home_team_dataaway_team_datarunsdisplay_namehome_team_nameaway_team_namevisitor_teamhome_team_scorevisitor_team_scorer   r   draw-r   r   Fr   r   Tr   r   r   rq   r   ro   odds_american_home	home_oddsodds_american_visitor	away_oddsrp   away_spreadrt   r   r   r   estimated_DO_NOT_BACKTESTr   pushoverunder
	homeScore	awayScorehomeQ1homeQ2homeQ3homeQ4awayQ1awayQ2awayQ3awayQ4winnerspreadCoveredtotalResultr   totalPoints)r   bdl_game_idrD   r   rE   homeTeamawayTeamscoresoddsr   result)	r   hashlibmd5encode	hexdigestr   r`   r  abs)#r   rD   rE   r  r   r  r  r  r  	home_name	away_name	home_data	away_datar   r   r)  rn   has_real_oddsembedded_oddsr   r   r   rq   r   r   r   r   ml_dataspread_dataaway_spread_val
total_datar1  	estimatedspread_coveredtotal_results#                                      r!   convert_gamerE  ,  s|    ~HH["-	xxR0XXlA.3!
XXlA.3!
HLQ\H]aeaiaijpaqDHH[$((62*>?Dwy	MM+y}}VY/OP	 $$[,2B2B692UV		%HH["-	xxR0HH-r2	HH-r2	]]61-2
]]61-2
15&1ADHHVR("-r	MM+.  R)--2O  RS\S`S`agSh  Rlpltlt  vF  HQ  mR	 $$[1  [\5E5En5U  [YeYiYijpYq  [uyu}u}  O  QZ  v[	 HH["-	xx3XX/38q
XX2A6;!
15&1ADHHVR("-r	MM+.R)--	2R	 $$[1X\5E5Efi5X	
*L*$F Q:? J	j	  kk'488D"%&a	{3::<ik"1G
 M xx !dhh|&<r2##$45##$45	 1 1- C HqIK	-++GQ7<1=E "	GGKKJ--,kk"67S7;;{;SGkk"9:Vgkk+>VG"w': $  mmH-)oom<O*#O4*l#.??3G#N #.??3JD#Q  $ ]]7+
#5U9UJ%":.
 -4S\$-4S\$%%7Gc"23T7Gc"23T#	
 "&,>	9	( C$$'2QVFNs;''''2QVFN#N *$!LJ&"L!L xx~##
 $+''
- g	K	Es   .W W WW"api_keyc                 @   | st        d       g S t        d       g }d}	 d| d}d|  d}t        r;t        j                  ||z   d	      }|j	                          |j                         }n}t        j                  j                  ||z         }t        j                  j                  |d	      5 }t        j                  |j                         j                               }d
d
d
       t        dt               d       d| d}	d|  d}
t        r;t        j                  |	|
z   d	      }|j	                          |j                         }n}t        j                  j                  |	|
z         }t        j                  j                  |d	      5 }t        j                  |j                         j                               }d
d
d
       t        dt               d       i }|D ]  }|j                  d      }|s|j                  dg       }|s,|d   }|j                  dg       D ci c]  }|d   |
 }}|j                  di       j                  dg       }|j                  di       j                  dg       }|j                  di       j                  dg       }|||dd||<    |D ]  }|j                  d      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  d       xs g }d}d}|D ]j  }|j                  d!      |k(  r t        |j                  d"d      xs d      }7|j                  d!      |k(  sLt        |j                  d"d      xs d      }l |dk(  r|dk(  r|s|j                  |i       }|j                  dg       }|j                  dg       }|j                  dg       }d
} d
}!d
}"d
}#d
}$|D ]N  }%|%j                  d!      |k(  r|%j                  d#      } )|%j                  d!      |k(  s>|%j                  d#      }!P |D ]9  }%|%j                  d!      |k(  s|%j                  d$      }"|%j                  d#      }#; |D ](  }%|%j                  d!      d%k(  s|%j                  d$      }$* | d
uxs |"d
u}&|j                  d&d'      d
d( }'||z   }(||z
  })|r|d
d) n9t        j                   d*|' d+| j#                               j%                         d
d) d,|'d-|'v sd.|'v rd/nd0||||d
d
d
d
d
d
d
d
d1
| |!|"|#|"r|" nd
|$|&rdnd2d3|&|)dkD  rd4n|)dk  rd5nd6|"r|)|" kD  nd
|$r|(|$kD  rd7n
|$r|(|$k  rd8nd
|)|(d9d:
}*|j'                  |*        t        d;t        |       d<t        |D +cg c]  }+|+j                  d=      s|+ c}+       d>       |S # 1 sw Y   xY w# 1 sw Y   xY wc c}w c c}+w # t(        $ r-},t        d?|,        dd
l}-|-j-                          Y d
},~,|S d
},~,ww xY w)@zFetch NFL historical games and odds from The Odds API.
    This is used because BallDontLie has no 2024 NFL odds data.
    Returns list of games in our betting format.
    u!     ⚠️ THE_ODDS_API_KEY not setz;  Fetching NFL from The Odds API (historical + upcoming)...r   z'https://api.the-odds-api.com/v4/sports/z/scoresr   z&daysFrom=3r1   r4   Nz    Got z games from scores APIr   z:&regions=us&markets=h2h,spreads,totals&oddsFormat=americanz games with oddsr   r   r   r   r   r   r   r   ru   r   )r   r   ru   r   	completedFr   r  r   r0  r   scorer   r   Overcommence_timer   r   r   znfl-r  r+   2024z2025-01r$   r%   r  r  )homeMoneylineawayMoneyline
homeSpreadhomeSpreadOdds
awaySpreadrq   r   r   r   r  r  r  r(  )
r   rD   r   rE   r.  r/  r0  r1  r   r2  z    Processed z NFL games with r   z having real oddsz    The Odds API error: )r[   r6   r7   r   r8   r9   r:   r;   r<   r=   r>   r?   r@   r_   r`   r3  r4  r5  r6  r   rc   	traceback	print_exc).rF  ri   rD   
scores_urlr   r   scores_datarB   rA   odds_urlodds_params	odds_resp	odds_dataodds_lookupogrn   r   r   r   r   r   r   ru   r   rH  r   r   r0  r  r  sr   r   r   r   r   r   r   r<  r  r   r   	convertedr   rj   rR  s.                                                 r!   fetch_nfl_from_the_odds_apir^    s   
 12		
GHE"ET>ugWM
G9K0<<
V 3R@D!!#))+K..((f)<=C''R'8 CH"jj)?)?)ABC 	[)**@AB =UG5I 	)cd X%;RHI&&(!(I..((K)?@C''R'8 AH JJx}}'='='?@	A 	Y((89:  	BffTlGVVL"5
#AB46FF9b4IJqqx{JGJ!++eR044ZDC%kk)R8<<ZLG$[[26:::rJF  ##*"("0	,K(	*   W	$DhhtnGe4Ii8Ii8IXXh'-2FJJ =55=I-!$QUU7A%6%;!!<JUU6]i/!$QUU7A%6%;!!<J	= Q:?9 $4I--r*CmmIr2G]]8R0F GGK#K -55=I-eeGnGUU6]i/eeGnG	-  655=I-"#%%.K'(uuW~$6
  155=F*"#%%.K1 $4/J;d3JM "5cr:I%
2L*,F &-gbqk'++YKqQZP[>\>c>c>e2f2p2p2rsutu2v!"(I"5i9O$UY%%!+!+"ddd"ddd	 &-%,"-&62=;,4(0=nC^  -(.
f6A:SY>IV{l%:t-8\K=W6hs  yE  HS  yS]d  Y]$#//I@ LL#oW	$r 	s5zl*:35?iaTUTYTYZgTh?i;j:kk|}~ LWC CA A KN @j (,-Ls   BW' 02W"B0W' 2W:W' ?W' W' -W:D'W' "B?W' ",W' =W' DW' W"/W"3W' WW' WW' '	X0"XXseasons_to_keeprequire_real_oddsc                    |dg}t         j                  |       }|st        d|         yt        dd        t        d| j                                 t        d        |j                  ddg      }g }i }|D ]^  }t        d| j                          d	| d
       t	        | |dd      }|r+|j                  |       t        dt        |       d|        ` |}	|}
i }|
j                         D ]  \  }}|j                  dd      }|j                  dd      j                         j                         }|j                  dd      j                         j                         }|sw|sz|s}| d| d| }|||<    |	rd|
rb|	dd D cg c]  }t        |j                  d             }}t        |
j                               dd }t        d|        t        d|        |	st        d| j                          d       yt        dt        |	       dt        |
       d       t        dt        |	       d       g }d}d}|	D ]  }	 |j                  d      }|
j                  |      }|sD|rA|j                  d       r|j                  d d      dd! nd}|j                  di       }|j                  d"|j                  di             }t        |t              r.|j                  d#d      j                         j                         n&t        |      j                         j                         }t        |t              r.|j                  d#d      j                         j                         n&t        |      j                         j                         }| d| d| }|j                  |      }|j                  d$|r|d%   nd      } t        || | |&      }!|!r,|r|!j                  d'      s|d(z  }|j!                  |!       n|d(z  } t        |D cg c]  }|j                  d'      s| c}      }#t        d+t        |       d,| d-| d.       t        d/|#        t        d0t        |      |#z
          t$        s| d1k(  r[t'        | |      }$d}%|$r9|D ]3  }|j                  d'      s|j                  d       s'|j                  d       t)        |j                  d2            t)        |j                  d3            f}|$j                  |      }&|&s|&j                  d4      |d5   d4<   |&j                  d6      |d5   d6<   |&j                  d7      |d5   d7<   |&j                  d8      |d5   d8<   |&j                  d9d:      |d5   d9<   |&j                  d;d:      |d5   d;<   |&j                  d<      |d5   d<<   d=|d5   d><   d?|d'<   |%d(z  }%6 |%rt        d@|% d       t*        |dA   z  }'g }(|'j-                         r	 t/        |'dB      5 })t1        j2                  |)      }*ddd       *D cg c]P  }|j                  d$      |v r;|j                  d       dCvr(|j                  d2      dCvr|j                  d3      dCvr|R }(}t        |*      t        |(      z
  }+|+dkD  rt        dD|+ dE       |(|z   }i },|D ]  }|j                  d      }-|-s||,|-<    t        |,j5                               }|j7                  dG H       t        dIt        |       dJ       t/        |'dK      5 })t1        j8                  ||)dLM       ddd       t        dN|'        |rgt        dO| j                          dP       |ddQ D ]  }|j                  d |j                  ddR            }|j                  d2t        |j                  d      t              r"|j                  di       j                  d#dR      ndR      }|j                  d3t        |j                  d"      t              r"|j                  d"i       j                  d#dR      ndR      }|j                  dSi       }.t        dT| d*| dU| dV|.j                  dWdX       dY|.j                  dZdX       d[        t        |      dQkD  rt        d\t        |      dQz
   d]       | t        |      t        |D cg c]:  }|j                  d'      s%|j                  d5i       j                  d>      d^k(  s9|< c}      t        |D cg c])  }|j                  d5i       j                  d>      d_k(  s(|+ c}      |d`S c c}w # t"        $ r0}"t        d)|j                  d       d*|"        |d(z  }Y d}"~"Vd}"~"ww xY wc c}w # 1 sw Y   xY wc c}w # t"        $ r}"t        dF|"        Y d}"~"d}"~"ww xY w# 1 sw Y   xY wc c}w c c}w )aa  Process a single sport - fetch, convert, and save.
    
    Args:
        sport: Sport code (nba, nfl, etc.)
        seasons_to_keep: List of season years to keep in the dataset
        require_real_odds: If True, only include games with real odds (recommended for backtesting)
    Nr$   rI   
<============================================================zProcessing r'   u   
📅 Fetching z season rO   r   )rF   rG   z  Got rL   r  r   r   r   :r   r   z)  DEBUG: Sample game IDs from games API: z'  DEBUG: Sample game IDs in odds dict: zNo z! games fetched. Check API access.u   
📊 Total: z games, real odds for r   z
Converting z games to betting format...r   r   r   r  r   rE   r   )r  r   rJ   z"  Warning: Failed to convert game r   z
Converted z games (skipped z incomplete, z without real odds)u     📈 Games with REAL odds: u%     ⚠️  Games with estimated odds: r+   r.  r/  r   r1  r   r   r   r   r   r   r   r   r   Tu1     ✅ Backfilled real odds from The Odds API for r(   r)Nr   r  zRemoved z games from old seasonsz'Warning: Could not read existing data: c                 &    | j                  dd      S )Nr   r   r   )r   s    r!   <lambda>zprocess_sport.<locals>.<lambda>=  s    vr!2 r#   )r   z
Total games in database: r   wr   indentz	Saved to z
Sample z games:r   r  r0    z @ r   r  ?r  r  rT   z
  ... and z more gamesr   r  )rD   total_gamesgames_with_real_oddsgames_with_estimated_oddsr'   )rZ   r   r[   r\   rk   r^   r_   itemsr   r   r   r   keysr   r   rE  r   rc   BACKFILL_ALL_SPORTSr   r"   DATA_DIRexistsopenr9   loadvaluessortdump)/rD   r_  r`  rd   r'   re   rv   rE   season_gamesri   real_odds_by_gameodds_by_matchupodds_idrY  r  r   r   r   r   sample_game_idssample_odds_idsbetting_gamesskippedskipped_no_oddsr   r-  r   r   r   r8  r9  matchup_keygame_seasonr]  rj   ro  fallbackfilledr   r(   existing_gamesfexisting_dataremovedby_idgidr0  s/                                                  r!   process_sportr    sW
    &e$Fw'(	Bxj/	K
'(	XJ jjTF+GIH  xxsCD&ufscR\*F3|,-[AB 	 E  O/557 -MM+r2	}}["-335;;=}}["-335;;=$Kqav.C#,OC - "5:2AY?3quuT{+??05578!<9/9JKL77HIJEKKM?"CDE	N3u:,&<SAR=S<TTZ
[\ 
M#e*%@
ABMGO 	((4.K)--k:I 9=&9IDHHVR0"5r	 HH["5	 HH^TXXk25NO	IST]_cIdIMM&"5;;=CCEjmnwjxj~j~  kA  kG  kG  kI	IST]_cIdIMM&"5;;=CCEjmnwjxj~j~  kA  kG  kG  kI	!*1YKqD+//<	 ((8GWR[NK$T5+SI$Y]]=-I#q(O$$Y/13< =QaAEE-<PQR	Js=)**:7)=Q`Paat
uv	)*>)?
@A	1#m2DG[2[1\
]^ eun.umD" 55'quuV}uuV}oaeeJ6G&H/Z[Z_Z_`jZkJlmLL%-.UU?-C&	/*-.UU?-C&	/**+%%*=&	,'*+%%*=&	,'./ee4Dd.K&	*+./ee4Dd.K&	*+)*{);&	+&&4&	(##'- !!" EfXVTU VM22KN	Ak3' -1 $		!-
 "/88H%8HHV$,AAHHZ(0EEHHZ(0EE	 N  -(3~+>>G{	)@AB .I E hhtnc
	
 U\\^$I NN2N3	'I'7q
9: 
k3	 *1		)Qq)* 
Ik]
#$ 	%++-01bqM 	sD+y)IJI88J\fgkgogop{g|  C  ]Db(A(E(Efi(X  JS  TD88J_ijnjrjr  tB  kC  EI  `J(D(H(HQZ([  PY  ZDXXh+FBykD6TF"VZZC5P4QQRSYS]S]^ijmSnRoopqr	s y>AJs9~12+>? 9~ #	$1QUU==QUVUZUZ[aceUfUjUjksUtx~U~Q$  !A%(Y  *B!%%PRBSBWBWX`Ba  fA  CA!  *B  &C U @V  	6txx~6FbLMqLG	  RJ- -  	A;A3?@@	A,* *( %@ *Bs   !e'F+e,<e,f(6f(f? f-/f? ;Af:-f? 4g$!:g1g1/)g6g6,	f%5%f  f%-f72f? ?	g!gg!$g.c                  ,   t        d       t        d       t        d       t        j                  dd       t        j                  dd       g d} | }t	        t
        j                        dkD  rt
        j                  d   j                         }|dk(  r| }n_|t        v r|g}nSt        d|        t        d	d
j                  t        j                                d       t        j                  d       dt
        j                  v xs dt
        j                  v }|rt        d       g }|D ]&  }t        |ddg|      }|s|j                  |       ( t        d |D              }t        d |D              }t        j                          j#                         dt        d |D              |||D 	cg c]  }	|	d   	 c}	t%        t'        d |D                    d|dkD  rdgng d	}
t        dz  }t)        |d      5 }t+        j,                  |
|d       d d d        t        d!d        t        d"       t        d        |D ]R  }	|	j/                  d#d      }|	j/                  d$d      }t        d%|	d   j1                          d&|	d'    d(| d)| d*	       T t        d%d+        t        d,|
d-    d.t	        |       d/       t        d0|        |dkD  rt        d1|        t        d2|        t        d3       y c c}	w # 1 sw Y   xY w)4zMain entry point.rc  z2BallDontLie Sports Data Fetcher - All Major SportsT)parentsexist_okr)   rJ   allrI   zAvailable: z, z, allz--require-real-oddsz-ruZ   
⚠️  REQUIRE_REAL_ODDS mode: Only games with verified sportsbook odds will be includedr$   r%   )r_  r`  c              3   @   K   | ]  }|j                  d d        yw)ro  r   Nrg  .0re  s     r!   	<genexpr>zmain.<locals>.<genexpr>  s     Lq!%% 6:L   c              3   @   K   | ]  }|j                  d d        yw)rp  r   Nrg  r  s     r!   r  zmain.<locals>.<genexpr>  s     QA!%% ;Q?Qr  zBallDontLie APIc              3   &   K   | ]	  }|d      yw)rn  N r  s     r!   r  zmain.<locals>.<genexpr>  s     <q!M*<s   rD   c              3   N   K   | ]  }|j                  d g       D ]  }|   yw)r'   Nrg  )r  re  r\  s      r!   r  zmain.<locals>.<genexpr>  s'     M!i8LM1AMAMs   #%z	2024-2025r   zUGames with 'estimated_DO_NOT_BACKTEST' odds source should NOT be used for backtesting)	
importedAtr   
totalGamesgamesWithRealOddsgamesWithEstimatedOddssportsr'   	dateRangewarningszmetadata.jsonri  r   rj  Nrb  SUMMARYro  rp  rl  r   rn  u    games (📈 u    real odds, ⚠️ z estimated)uZ   ──────────────────────────────z	  TOTAL: r  z games across z sportsu4     📈 Games with REAL odds (safe for backtesting): u7     ⚠️  Games with ESTIMATED odds (DO NOT BACKTEST): z
Metadata saved to: u
   
✅ Done!)r[   rt  mkdir	CACHE_DIRr_   sysargvr   rZ   r   rr  exitr  r   sumr   now	isoformatr   r   rv  r9   rz  r   r\   )
all_sportssports_to_processargr`  resultsrD   r2  total_real_oddstotal_estimatedre  metadatametadata_filer  realests                  r!   mainr  \  s   	(O	
>?	(O NN4$N/OOD4O0 .J"
388}qhhqk!%< *L !$OC5)*K		,*;*;*= >?uEFHHQK .9MTSXX=Mkl G" #utTlVghNN6"# LGLLOQQQO lln..0#<G<<,"1'./!1W:/MwMMN  rA  DE  rElm  KM
H .M	mS	! )Q		(Aa() 
Bxj/	)	XJ suu+Q/ee/31W:##%&b=)9(:-vM`ad`eepqrs 
Bzl
	Ih|,-^CL>
QR	@@Q
RSGGXYZ	!-
12	-/ 0) )s   7LL

L__main__)r   r   )r   2   )N)NF)/__doc__osr  r  environr   r   rs  r9   ra   r3  r   r	   pathlibr
   r7   r6   ImportErrorurllib.requestr:   urllib.errorr]   __file__parentrt  r  r5   r   r   r"   rZ   r   rC   r`   r   rk   rz   tupler   r   r  rE  r^  boolr  r  __name__r  r#   r!   <module>r     s   
 
 -8  IbjjnnMdfh>i>o>o>q  vH  ?H     ( L ,>  ''&09<N!!((614GG	 **...0V
W::>>"4b9 # # $ $<,  $<,  $<,  6,!08c 8d 873 7 7s 7S 7[_ 7tB3 B Bs BS BZ^ BNK%3 K%5 K%\Ds D4 DD DPl# lS l l l^zt zC z z zQU z~d d dPF Ft Ft FRFR zF I$  Ls   "F FF