#!/usr/bin/env python3 """ The Odds API - Game Odds Scraper Fetches spreads, totals (O/U), and moneylines for all major sports """ import requests import psycopg2 from datetime import datetime, timezone, timedelta ODDS_API_BASE = 'https://api.the-odds-api.com/v4' # Expanded sport key mapping - 30+ sports/leagues SPORTS = { # Major US Sports 'nba': 'basketball_nba', 'nfl': 'americanfootball_nfl', 'mlb': 'baseball_mlb', 'nhl': 'icehockey_nhl', 'ncaaf': 'americanfootball_ncaaf', 'ncaab': 'basketball_ncaab', 'mls': 'soccer_usa_mls', # MMA/Boxing 'mma': 'mma_mixed_martial_arts', 'boxing': 'boxing_boxing', # European Soccer - Top 5 Leagues 'epl': 'soccer_epl', 'laliga': 'soccer_spain_la_liga', 'seriea': 'soccer_italy_serie_a', 'bundesliga': 'soccer_germany_bundesliga', 'ligue1': 'soccer_france_ligue_one', # UEFA Competitions 'ucl': 'soccer_uefa_champs_league', 'uel': 'soccer_uefa_europa_league', # English Football Pyramid 'championship': 'soccer_efl_champ', 'league1': 'soccer_england_league1', 'league2': 'soccer_england_league2', 'facup': 'soccer_fa_cup', 'eflcup': 'soccer_england_efl_cup', # Other European Leagues 'eredivisie': 'soccer_netherlands_eredivisie', 'ligamx': 'soccer_mexico_ligamx', 'primeira': 'soccer_portugal_primeira_liga', 'spl': 'soccer_spl', 'super_lig': 'soccer_turkey_super_league', # South American Soccer 'brazil_a': 'soccer_brazil_campeonato', 'argentina': 'soccer_argentina_primera_division', 'libertadores': 'soccer_conmebol_copa_libertadores', # Other Hockey 'ahl': 'icehockey_ahl', 'shl': 'icehockey_sweden_hockey_league', # Other Basketball 'euroleague': 'basketball_euroleague', 'nbl': 'basketball_nbl', # Tennis Grand Slams 'atp_aus_open': 'tennis_atp_aus_open_singles', 'wta_aus_open': 'tennis_wta_aus_open_singles', # Rugby 'nrl': 'rugbyleague_nrl', 'six_nations': 'rugbyunion_six_nations', # Australian Rules 'afl': 'aussierules_afl', # Cricket 'big_bash': 'cricket_big_bash', 'cricket_t20': 'cricket_international_t20', } # Priority sports to fetch more frequently (main leagues) PRIORITY_SPORTS = [ 'nba', 'nfl', 'mlb', 'nhl', 'ncaaf', 'ncaab', 'epl', 'laliga', 'seriea', 'bundesliga', 'ligue1', 'ucl', 'mma' ] # Secondary sports (fetch less frequently to conserve API calls) SECONDARY_SPORTS = [ 'mls', 'boxing', 'championship', 'facup', 'uel', 'eredivisie', 'ligamx', 'primeira', 'ahl', 'euroleague', 'brazil_a', 'argentina' ] def load_db_url(): with open('/var/www/html/eventheodds/.env', 'r') as f: for line in f: if line.startswith('SPORTS_DATABASE_URL='): return line.split('=', 1)[1].strip().split('?')[0] return '' def load_api_key(): with open('/var/www/html/eventheodds/.env', 'r') as f: for line in f: if line.startswith('THE_ODDS_API_KEY='): return line.split('=', 1)[1].strip() return None def check_api_quota(): """Check remaining API quota""" api_key = load_api_key() if not api_key: return None url = f'{ODDS_API_BASE}/sports' try: resp = requests.get(url, params={'apiKey': api_key}, timeout=10) remaining = resp.headers.get('x-requests-remaining', 'unknown') used = resp.headers.get('x-requests-used', 'unknown') return {'remaining': remaining, 'used': used} except: return None def fetch_odds(sport_key, markets='h2h,spreads,totals', regions='us'): """Fetch odds from The Odds API""" api_key = load_api_key() if not api_key: print('No API key found') return [] url = f'{ODDS_API_BASE}/sports/{sport_key}/odds' params = { 'apiKey': api_key, 'regions': regions, 'markets': markets, 'oddsFormat': 'american', } print(f'Fetching odds for {sport_key}...') try: resp = requests.get(url, params=params, timeout=30) if resp.status_code == 401: print(' Invalid API key') return [] if resp.status_code == 404: print(f' Sport {sport_key} not found or no events') return [] if resp.status_code == 429: print(' Rate limited - quota exceeded') return [] if resp.status_code != 200: print(f' Status: {resp.status_code}') return [] data = resp.json() print(f' Found {len(data)} events') remaining = resp.headers.get('x-requests-remaining', 'unknown') print(f' API requests remaining: {remaining}') return data except Exception as e: print(f'Error: {e}') return [] def fetch_historical_odds(sport_key, date_str, markets='h2h,spreads,totals'): """Fetch historical odds (requires paid plan)""" api_key = load_api_key() if not api_key: return [] url = f'{ODDS_API_BASE}/historical/sports/{sport_key}/odds' params = { 'apiKey': api_key, 'regions': 'us', 'markets': markets, 'oddsFormat': 'american', 'date': date_str, } print(f'Fetching historical odds for {sport_key} on {date_str}...') try: resp = requests.get(url, params=params, timeout=30) if resp.status_code != 200: print(f' Status: {resp.status_code} (historical may require paid plan)') return [] return resp.json().get('data', []) except Exception as e: print(f'Error: {e}') return [] def safe_float(val): try: if val is None: return None return float(val) except: return None def ingest_odds(leagues=None, priority_only=False): """Main ingest function""" db_url = load_db_url() if not db_url: raise RuntimeError('SPORTS_DATABASE_URL not set') conn = psycopg2.connect(db_url) cur = conn.cursor() stats_added = 0 cur.execute(''' CREATE TABLE IF NOT EXISTS "GameOdds" ( id BIGSERIAL PRIMARY KEY, league TEXT NOT NULL, "gameId" TEXT NOT NULL, "gameDate" TIMESTAMPTZ, "homeTeam" TEXT, "awayTeam" TEXT, "bookmaker" TEXT, "market" TEXT NOT NULL, "lineValue" FLOAT, "homeOdds" INT, "awayOdds" INT, "overOdds" INT, "underOdds" INT, "fetchedAt" TIMESTAMPTZ DEFAULT NOW(), raw JSONB, source TEXT DEFAULT 'theoddsapi', UNIQUE(league, "gameId", "bookmaker", "market") ) ''') conn.commit() sql = ''' INSERT INTO "GameOdds" (league, "gameId", "gameDate", "homeTeam", "awayTeam", "bookmaker", "market", "lineValue", "homeOdds", "awayOdds", "overOdds", "underOdds", raw) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (league, "gameId", "bookmaker", "market") DO UPDATE SET "lineValue" = EXCLUDED."lineValue", "homeOdds" = EXCLUDED."homeOdds", "awayOdds" = EXCLUDED."awayOdds", "overOdds" = EXCLUDED."overOdds", "underOdds" = EXCLUDED."underOdds", "fetchedAt" = NOW(), raw = EXCLUDED.raw ''' if leagues: target_leagues = leagues elif priority_only: target_leagues = PRIORITY_SPORTS else: target_leagues = PRIORITY_SPORTS + [s for s in SECONDARY_SPORTS if s not in PRIORITY_SPORTS] quota = check_api_quota() if quota: print(f"API Quota - Remaining: {quota['remaining']}, Used: {quota['used']}") total_events = 0 for league in target_leagues: sport_key = SPORTS.get(league) if not sport_key: print(f'Unknown league: {league}') continue events = fetch_odds(sport_key) league_count = 0 for event in events: game_id = event.get('id', '') commence_time = event.get('commence_time', '') home_team = event.get('home_team', '') away_team = event.get('away_team', '') try: game_date = datetime.fromisoformat(commence_time.replace('Z', '+00:00')) except: game_date = datetime.now(timezone.utc) for bookmaker in event.get('bookmakers', []): book_key = bookmaker.get('key', '') for market in bookmaker.get('markets', []): market_key = market.get('key', '') outcomes = market.get('outcomes', []) line_value = None home_odds = None away_odds = None over_odds = None under_odds = None if market_key == 'h2h': for outcome in outcomes: if outcome.get('name') == home_team: home_odds = safe_float(outcome.get('price')) elif outcome.get('name') == away_team: away_odds = safe_float(outcome.get('price')) elif market_key == 'spreads': for outcome in outcomes: if outcome.get('name') == home_team: line_value = safe_float(outcome.get('point')) home_odds = safe_float(outcome.get('price')) elif outcome.get('name') == away_team: away_odds = safe_float(outcome.get('price')) elif market_key == 'totals': for outcome in outcomes: if outcome.get('name') == 'Over': line_value = safe_float(outcome.get('point')) over_odds = safe_float(outcome.get('price')) elif outcome.get('name') == 'Under': under_odds = safe_float(outcome.get('price')) try: import json cur.execute(sql, ( league, game_id, game_date, home_team, away_team, book_key, market_key, line_value, int(home_odds) if home_odds else None, int(away_odds) if away_odds else None, int(over_odds) if over_odds else None, int(under_odds) if under_odds else None, json.dumps({'event': event, 'market': market}) )) stats_added += 1 league_count += 1 except Exception as e: conn.rollback() conn.commit() total_events += len(events) print(f' {league.upper()} odds added: {league_count}') cur.close() conn.close() print(f'') print(f'✅ Game Odds ingested: {stats_added} total records from {total_events} events') return {'success': True, 'stats_added': stats_added, 'events': total_events} if __name__ == '__main__': import sys priority_only = '--priority' in sys.argv leagues = None for arg in sys.argv[1:]: if not arg.startswith('--'): leagues = arg.split(',') break print("=" * 60) print("THE ODDS API - EXPANDED SPORTS SCRAPER") print(f"Time: {datetime.now(timezone.utc).isoformat()}") print(f"Configured sports: {len(SPORTS)}") if priority_only: print(f"Mode: Priority sports only ({len(PRIORITY_SPORTS)} leagues)") print("=" * 60) ingest_odds(leagues, priority_only=priority_only)