#!/usr/bin/env python3 """ Apify Sportsbook Odds Scraper Integration Fetches odds from multiple sportsbooks via Apify actors Includes opening lines, current lines, and best line indicators """ import requests import psycopg2 import json from datetime import datetime, timezone import time import os APIFY_API_BASE = 'https://api.apify.com/v2' SPORTSBOOK_ACTOR_ID = 'oGSyv8PIttFIMoHfz' # sportsbook-odds-scraper # Supported leagues APIFY_LEAGUES = ['NBA', 'NFL', 'NHL', 'College-Basketball', 'College-Football', 'UFC'] def load_db_url(): with open('/var/www/html/eventheodds/.env', 'r') as f: for line in f: if line.startswith('SPORTS_DATABASE_URL='): return line.split('=', 1)[1].strip().split('?')[0] return '' def load_apify_key(): # Check env var first key = os.environ.get('APIFY_API_KEY') if key: return key # Fall back to .env file with open('/var/www/html/eventheodds/.env', 'r') as f: for line in f: if line.startswith('APIFY_API_KEY='): return line.split('=', 1)[1].strip() return None def run_apify_actor(league, api_key): """Run the sportsbook odds scraper actor""" url = f'{APIFY_API_BASE}/acts/{SPORTSBOOK_ACTOR_ID}/runs' params = {'waitForFinish': 180} # Wait up to 3 minutes headers = { 'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json' } body = {'league': league} print(f' Running Apify actor for {league}...') try: resp = requests.post(url, params=params, headers=headers, json=body, timeout=200) if resp.status_code != 201: print(f' Error: {resp.status_code} - {resp.text[:200]}') return None data = resp.json().get('data', {}) return data.get('defaultDatasetId') except Exception as e: print(f' Error running actor: {e}') return None def fetch_dataset(dataset_id, api_key): """Fetch results from Apify dataset""" url = f'{APIFY_API_BASE}/datasets/{dataset_id}/items' headers = {'Authorization': f'Bearer {api_key}'} try: resp = requests.get(url, headers=headers, timeout=60) if resp.status_code != 200: print(f' Error fetching dataset: {resp.status_code}') return [] return resp.json() except Exception as e: print(f' Error: {e}') return [] def safe_int(val): """Convert odds string to int""" if val is None: return None try: # Remove + sign if present val_str = str(val).replace('+', '') return int(float(val_str)) except: return None def safe_float(val): """Convert to float""" if val is None: return None try: val_str = str(val).replace('+', '') return float(val_str) except: return None def ingest_apify_odds(leagues=None): """Main ingest function""" api_key = load_apify_key() if not api_key: print('Error: APIFY_API_KEY not found') return {'success': False, 'error': 'No API key'} db_url = load_db_url() if not db_url: print('Error: SPORTS_DATABASE_URL not found') return {'success': False, 'error': 'No DB URL'} conn = psycopg2.connect(db_url) cur = conn.cursor() stats_added = 0 # Ensure GameOdds table exists with additional columns for Apify data cur.execute(''' ALTER TABLE "GameOdds" ADD COLUMN IF NOT EXISTS "openingLineValue" FLOAT, ADD COLUMN IF NOT EXISTS "openingHomeOdds" INT, ADD COLUMN IF NOT EXISTS "openingAwayOdds" INT, ADD COLUMN IF NOT EXISTS "openingOverOdds" INT, ADD COLUMN IF NOT EXISTS "openingUnderOdds" INT, ADD COLUMN IF NOT EXISTS "isBestLine" BOOLEAN DEFAULT FALSE, ADD COLUMN IF NOT EXISTS "source" TEXT DEFAULT 'odds-api' ''') conn.commit() sql = ''' INSERT INTO "GameOdds" (league, "gameId", "gameDate", "homeTeam", "awayTeam", "bookmaker", "market", "lineValue", "homeOdds", "awayOdds", "overOdds", "underOdds", "openingLineValue", "openingHomeOdds", "openingAwayOdds", "openingOverOdds", "openingUnderOdds", "isBestLine", "source", raw) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (league, "gameId", "bookmaker", "market") DO UPDATE SET "lineValue" = EXCLUDED."lineValue", "homeOdds" = EXCLUDED."homeOdds", "awayOdds" = EXCLUDED."awayOdds", "overOdds" = EXCLUDED."overOdds", "underOdds" = EXCLUDED."underOdds", "openingLineValue" = EXCLUDED."openingLineValue", "openingHomeOdds" = EXCLUDED."openingHomeOdds", "openingAwayOdds" = EXCLUDED."openingAwayOdds", "openingOverOdds" = EXCLUDED."openingOverOdds", "openingUnderOdds" = EXCLUDED."openingUnderOdds", "isBestLine" = EXCLUDED."isBestLine", "source" = EXCLUDED."source", "fetchedAt" = NOW(), raw = EXCLUDED.raw ''' target_leagues = leagues or APIFY_LEAGUES for league in target_leagues: print(f'\n=== Processing {league} ===') # Map Apify league names to our league codes league_map = { 'NBA': 'nba', 'NFL': 'nfl', 'NHL': 'nhl', 'College-Basketball': 'ncaab', 'College-Football': 'ncaaf', 'UFC': 'ufc', 'UCL': 'ucl' } league_code = league_map.get(league, league.lower()) # Run actor dataset_id = run_apify_actor(league, api_key) if not dataset_id: print(f' No dataset returned for {league}') continue # Fetch results games = fetch_dataset(dataset_id, api_key) print(f' Found {len(games)} games') league_count = 0 for game in games: home_team = game.get('homeTeam', {}) away_team = game.get('awayTeam', {}) home_name = f"{home_team.get('location', '')} {home_team.get('nickName', '')}".strip() away_name = f"{away_team.get('location', '')} {away_team.get('nickName', '')}".strip() if not home_name or not away_name: continue # Generate game ID from teams and date scheduled = game.get('scheduledTime', '') try: game_date = datetime.fromisoformat(scheduled.replace('Z', '+00:00')) except: game_date = datetime.now(timezone.utc) game_id = f"apify_{home_team.get('abbr', 'UNK')}_{away_team.get('abbr', 'UNK')}_{game_date.strftime('%Y%m%d')}" for odds_data in game.get('odds', []): bookmaker = odds_data.get('sportsbook', 'unknown') # Skip consensus (aggregated) odds if bookmaker == 'consensus': continue # Process moneyline ml = odds_data.get('moneyLine', {}) if ml: try: cur.execute(sql, ( league_code, game_id, game_date, home_name, away_name, bookmaker, 'h2h', None, # lineValue not applicable for h2h safe_int(ml.get('currentHomeOdds')), safe_int(ml.get('currentAwayOdds')), None, None, # over/under not applicable None, # opening line value safe_int(ml.get('openingHomeOdds')), safe_int(ml.get('openingAwayOdds')), None, None, # opening over/under ml.get('isBestHomeLine', False) or ml.get('isBestAwayLine', False), 'apify', json.dumps(ml) )) stats_added += 1 league_count += 1 except Exception as e: conn.rollback() # Process spread spread = odds_data.get('pointSpread', {}) if spread: try: cur.execute(sql, ( league_code, game_id, game_date, home_name, away_name, bookmaker, 'spreads', safe_float(spread.get('currentHomeHandicap')), safe_int(spread.get('currentHomeOdds')), safe_int(spread.get('currentAwayOdds')), None, None, safe_float(spread.get('openingHomeHandicap')), safe_int(spread.get('openingHomeOdds')), safe_int(spread.get('openingAwayOdds')), None, None, spread.get('isBestHomeLine', False) or spread.get('isBestAwayLine', False), 'apify', json.dumps(spread) )) stats_added += 1 league_count += 1 except Exception as e: conn.rollback() # Process totals (over/under) ou = odds_data.get('overUnder', {}) if ou: try: cur.execute(sql, ( league_code, game_id, game_date, home_name, away_name, bookmaker, 'totals', safe_float(ou.get('currentTotal')), None, None, # home/away not applicable safe_int(ou.get('currentOverOdd')), safe_int(ou.get('currentUnderOdd')), safe_float(ou.get('openingTotal')), None, None, safe_int(ou.get('openingOverOdd')), safe_int(ou.get('openingUnderOdd')), ou.get('isBestOverLine', False) or ou.get('isBestUnderLine', False), 'apify', json.dumps(ou) )) stats_added += 1 league_count += 1 except Exception as e: conn.rollback() conn.commit() print(f' {league} odds added: {league_count}') # Rate limit between leagues time.sleep(2) cur.close() conn.close() print(f'\n✅ Apify Odds ingested: {stats_added} total records') return {'success': True, 'stats_added': stats_added} if __name__ == '__main__': import sys leagues = sys.argv[1].split(',') if len(sys.argv) > 1 else None ingest_apify_odds(leagues)