
    ozi>~                        d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
m
Z
mZmZ ddlmZmZmZ dZdZdZdZi d	d
dd
dd
dddddddddddddddddddddddddddd d d!d!d"d"d#d#d
ddd
ddd
d$Zd%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4Zg d5Zd6 Zd7 Zd8 Zd9 ZdKd:Zd; ZdLd<Zd= Zd> Z d? Z!d@ Z"dA Z#dBedCe$fdDZ%dBedCe&fdEZ'dMdBedCe&fdFZ(dNdBedCe&fdGZ)dOdHZ*dI Z+e,dJk(  r e+        yy)Pa  
OddsJam Scraper with Auto-2FA, Historical Backfill, and Daily Updates
Scrapes arbitrage, positive EV, and historical results from OddsJam.

Usage:
  python scrape_oddsjam.py                    # Daily scrape (live arb + EV)
  python scrape_oddsjam.py --backfill         # Backfill historical results from 2025
  python scrape_oddsjam.py --backfill --days 30  # Backfill last 30 days
  python scrape_oddsjam.py --results-only     # Only scrape results/history
    N)datetimetimezone	timedelta)sync_playwrightPageBrowserzadmin@cashhive.aiz/home/admin/Maildirz/var/www/html/eventheodds/.envz9/var/www/html/eventheodds/data/oddsjam_scraper_state.jsonBETBetMGMBETMGMMGMGAME
DraftKingsDK
DRAFTKINGSDUALz	Hard RockHARDROCKHRSITEFanDuelFDFANDUELBALJz	Bally BetBALLYCAESARSCaesarsCZR	POINTSBET	PointsBet	BetRiverszESPN BetPinnacle)PB	BETRIVERSBRESPNESPNBETPINNACLEPINHOUNYKCHADALATLINDMINnbancaabncaawnflncaafnhlmlbwnbamlseplsoccerufcmmatennisgolf)NBANCAABNCAAWNFLNCAAFNHLMLBWNBAMLSEPLSOCCERUFCMMATENNISGOLF)&zPlayer PointsPlayer AssistszPlayer ReboundszPlayer BlockszPlayer StealszPlayer ThreeszPlayer 3-Pointers MadezPlayer Points + ReboundszPlayer Points + AssistszPlayer Rebounds + Assistsz"Player Points + Rebounds + AssistszPlayer Double DoublezPlayer Triple DoublezPlayer Passing YardszPlayer Rushing YardszPlayer Receiving YardszPlayer Passing TDszPlayer Rushing TDszPlayer ReceptionszPlayer Anytime TDzPlayer First TDzPlayer HitszPlayer RunszPlayer RBIszPlayer Total BaseszPlayer StrikeoutszPlayer Home RunszPlayer GoalsrM   zPlayer ShotszPlayer SaveszTotal PointsSpread	Moneylinez
Over/UnderzFirst Half SpreadzFirst Half TotalzFirst Quarter Spreadc                  j   	 t        t              5 } | D ]d  }|j                  d      s|j                         j	                  dd      d   }|j                  d      j	                  d      d   c cddd       S  	 ddd       y# 1 sw Y   yxY w# t
        $ r}t        d|        Y d}~yd}~ww xY w)	z'Load sports database URL from .env filezSPORTS_DATABASE_URL==   "?r   NzError loading DB URL: )openDB_URL_FILE
startswithstripsplit	Exceptionprint)flineurles       3/var/www/html/eventheodds/scripts/scrape_oddsjam.pyload_db_urlra   c   s    ,+ 	8! 8??#9:**,,,S!4Q7C99S>//4Q77		8 	88	8 	8   ,&qc*++,sF   B BAB0	B :B<B B
B B 	B2B--B2c                      	 t        t              5 } t        j                  |       cddd       S # 1 sw Y   yxY w#  ddddcY S xY w)z.Load scraper state (last successful run, etc.)Nr   )last_runlast_backfill_datetotal_records)rU   
STATE_FILEjsonload)r\   s    r`   
load_stateri   p   sF    R* 	 99Q<	  	  	 R qQQs   < 0	< 9< < Ac                 8   	 t        j                  t         j                  j                  t              d       t        t        d      5 }t        j                  | |d       ddd       y# 1 sw Y   yxY w# t        $ r}t        d|        Y d}~yd}~ww xY w)zSave scraper stateT)exist_okw   )indentNzError saving state: )
osmakedirspathdirnamerf   rU   rg   dumprZ   r[   )stater\   r_   s      r`   
save_stateru   y   su    *
BGGOOJ/$?*c" 	*aIIeQq)	* 	* 	* *$QC())*s6   AA8 
A,#A8 ,A51A8 5A8 8	BBBc                     d| j                         vryd| j                         vry| j                  d      }|D ],  }|j                         }t        j                  d|      s*|c S  y)z#Extract 2FA code from email contentzoddsjam.comNzverification code
z^\d{6}$)lowerrY   rX   rematch)contentlinesr]   s      r`   get_oddsjam_code_from_contentr}      sd    GMMO+'--/1MM$E zz|88J%K     c                    t        d       t        j                         }t        j                         |z
  |k  rut        j                  d       	 dt         dt         d}t	        j
                  |ddd      }|j                  j                         j                  d      D cg c]#  }|j                         s|j                         % }}|D ]  }t	        j
                  d	| d
ddd      }	 t        |j                  j                               }	|	| k  rJt	        j
                  d| d
ddd      }
t        |
j                        }|st        d|        |c S  	 t        t        j                         |z
        }t        d| d       t        j                         |z
  |k  ruyc c}w #  Y xY w# t        $ r}t        d|        Y d}~td}~ww xY w)z&Wait for 2FA code to arrive in maildirz  Waiting for 2FA code...   z
sudo find z/new z!/cur -type f -mmin -5 2>/dev/nullT)shellcapture_outputtextrw   zstat -c %Y "rS   z
sudo cat "z  Found 2FA code: z	  Error: Nz  Still waiting... (zs))r[   timesleepMAILDIR_PATH
subprocessrunstdoutrX   rY   floatr}   rZ   int)min_timestamptimeout
start_timecmdresultr\   filesfilepathstat_resultmtime
cat_resultcoder_   elapseds                 r`   wait_for_new_coder      s   	%'J
))+

"W
,

1	#|nE,?`aC^^CtDtTF(.(;(;(=(C(CD(IW1QWWYQWWYWEW!  (nn|H:Q-Gtdhost!+"4"4":":"<=E =('^^j
!,DDaelpq
4Z5F5FG.tf56K " diikJ./$WIR013 ))+

"W
,4 + X  	#IaS/""	#sU   AF= &F1<F1$F= 3#F6:F= F= #F= 1F= 6F:8F= =	GGGc                    | syg d}|D ](  }	 t        j                  | j                         |      c S  t        j                  d|       }|rw	 |j                  d      }t        |j                  d            }t        |j                  d            }dddddd	d
dddddd}|j                  |dd d      }t        |||      S y#  Y xY w#  Y yxY w)z+Parse game date string into datetime objectN)z%a, %b %d, %Y %I:%M %pz%a, %b %d %Y %I:%M %pz%b %d, %Y %I:%M %pz%Y-%m-%d %H:%M:%Sz%Y-%m-%dz%m/%d/%Yz'(\w+),?\s+(\w+)\s+(\d{1,2}),?\s+(\d{4})rm         rR   r            	   
         )JanFebMarAprMayJunJulAugSepOctNovDec)r   strptimerX   ry   searchgroupr   get)	date_strformatsfmt
date_match	month_strdayyearmonthsmonths	            r`   parse_game_dater      s    G  	$$X^^%5s;; ExPJ		"((+Ij&&q)*Cz''*+Dq1Qqq2bQSUFJJy!}a0ED%-- #		s   #CA5C C	Cc                    g }| j                  d      }d}|t        |      k  rE||   j                         }t        j                  d|      }|rt        |j                  d            }|g |d}|dz   }	|	t        |      k  rd|	|dz   k  r\||	   j                         }
t        j                  d|
      rn2|
r|d   j                  |
       |	dz  }	|	t        |      k  r	|	|dz   k  r\|d   }t        |      D ].  \  }t        fd	d
D              s|d<   t              |d<    n t        |      D ]  \  }dv rR|d<   j                  d      }t        |      dk(  r,|d   j                         |d<   |d   j                         |d<    nXdv sa|d<   j                  d      }t        |      dk(  r,|d   j                         |d<   |d   j                         |d<    n t        |      D ]  \  }dv sj                  d      }t        |      dk\  r^|d   j                         |d<   |d   j                         j                         }t        j                  ||j                               |d<    n t        |      D ]=  \  }t        D ])  }|j                         j                         v s$||d<    n d|v s= n t        |      D ]T  \  }j!                  d      sj!                  d      s)|dkD  r&||dz
     }|t        vr|j!                  d      s||d<    n t        |      D ]}  \  }t        j                  d      }t        j                  d      }|r$d|d<   t        |j                  d            |d<    n(|s[d |d<   t        |j                  d            |d<    n t        |      D ]<  \  }t        j                  d!      }|st#        |j                  d            |d"<    n t        |      D ]+  \  }j                         }|t$        v st$        |   |d#<    n d|v rd"|v r|j                  |       |	}n|dz  }|t        |      k  rE|S )$z4Parse the raw text from OddsJam into structured datarw   r   z^(\d+\.\d+)%$rR   )edge_pct	raw_linessource_type2   z^\d+\.\d+%$r   c              3   @   K   | ]  }j                  |        y wNrW   ).0r   ls     r`   	<genexpr>z"parse_odds_text.<locals>.<genexpr>   s     mSq||C(m   zThu,zFri,zSat,zSun,zMon,zTue,zWed,game_date_str	game_date vs matchuprm   	away_team	home_teamz @    •sportleaguemarketzOver zUnder )r>   rA   rC   rD   player_name^Over (\d+\.?\d*)$^Under (\d+\.?\d*)$overside
line_valueunder^([+-]\d+)$oddsbook)rY   lenrX   ry   rz   r   r   append	enumerateanyr   upperLEAGUE_MAPPINGr   rx   MARKET_TYPESrW   r   BOOK_MAPPING)r   r   opportunitiesr|   ir]   
edge_matchedgeoppj	next_linerawidxteamsparts
league_rawmtpotential_name
over_matchunder_match
odds_matchupper_lr   s                         @r`   parse_odds_textr      s   MJJtE	A
c%j.Qx~~ XX.5
))!,-D  $"[QC AAc%j.QRZ!!HNN,	88NI6$++I6Q c%j.QRZ k"C $C. Qm4lmm+,C('6q'9C$	 $C. QQ;%&C	NGGFOE5zQ+08>>+;K(+08>>+;K(aZ%&C	NGGENE5zQ+08>>+;K(+08>>+;K(" $C. QA:GGENE5zQ',Qx~~'7G%*1X^^%5%;%;%=
(6(:(::zGWGWGY(ZH $C. Q& BxxzQWWY.(*H s? $C. Q<<(ALL,BQw),S1W)=nF_F_`|F}1?C. $C. 
QXX&;Q?
 hh'=qA"(CK(-j.>.>q.A(BC%")CK(-k.?.?.B(CC%
 $C. QXXna8
"%j&6&6q&9":CK	 $C. Q'')l*".w"7CK	 36S=$$S)AFA] c%j.` r~   c                 D   g }| j                  d      }d}d}|t        |      k  r||   j                         dv rL|rFj                         |d<   |j	                  d      r"|j	                  d      r|j                  |       dg i}t        j                  d	      }|rH|r"|j	                  d      r|j                  |       t        |j                  d
            dv rdndg d}|r|d   j                         t        fddD              r|d<   t              |d<   dv rP|d<   j                  d      }t        |      dk(  r,|d   j                         |d<   |d
   j                         |d<   dv rgj                  d      }t        |      dk\  rH|d
   j                         j                         }t        j	                  ||j                               |d<   t        D ])  }	|	j                         j                         v s$|	|d<    n t        j                  d      }
t        j                  d      }|
r"d|d<   t        |
j                  d
            |d<   |r"d|d<   t        |j                  d
            |d<   t        j                  d       }|rt        |j                  d
            |d<   j                         }|t         v rt         |   |d!<   |d
z  }|t        |      k  r|r"|j	                  d      r|j                  |       |S )"z"Parse historical results page textrw   Nr   )WonLostPushWINLOSSPUSHoutcomer   r   r   z^(\d+\.\d+)%\s*(EV|ARB)?$rR   EVpositive_ev	arbitrage)r   r   r   c              3   @   K   | ]  }j                  |        y wr   r   )r   r   r]   s     r`   r   z%parse_results_page.<locals>.<genexpr>t  s     lC4??3'lr   r   r   r   r   r   rm   r   r   r   r   r   r   r   r   r   r   r   r   )rY   r   rX   r   r   r   ry   rz   r   r   r   r   r   rx   r   r   r   )r   resultsr|   current_resultr   r   r   r   r   r   r   r   r   r   r]   s                 @r`   parse_results_pager  R  s	   GJJtEN	A
c%j.Qx~~ AA,0JJLy)!%%h/N4F4Fv4NNN>2)2.N XX:DA
."4"4X">~.!*"2"21"5604}+N ;'..t4 l3kll26/.=d.C{+ ~,0y)

6*u:?27(..2BN;/27(..2BN;/ }

5)u:?!&q!1!7!7!9J/=/A/A*jN^N^N`/aN8, # 88:-/1N8, "7>J((#94@K)/v&/4Z5E5Ea5H/I|,)0v&/4[5F5Fq5I/J|, .$7J),Z-=-=a-@)Av& jjlG,&)5g)>v&	QO c%j.R .,,X6~&Nr~   c                    	 | j                         }|j                  dd      }t        |t              r2t        j                  |j                         |j                               }|j                  dd      }|j                  dd      }|j                         j                  dd       d| }|j                  d      |j                  d	      |j                  d
      |j                  d      |j                  d      |j                  d      d}|j                  d||j                  dd      |||j                  dd      |j                  dd      dt        j                  |      |j                  d      xs# t        j                  t        j                        f	       | j                          |j                   dkD  S # t"        $ r(}t%        d|        | j'                          Y d}~yd}~ww xY w)z7Store a player prop opportunity in PlayerPropLine tabler   unknownr   r     _r   r   r   r   r   r   )r   r   r   r   r   r   a  
            INSERT INTO "PlayerPropLine"
            (league, "playerExternalId", "propType", market, "lineValue", "oddsAmerican", vendor, raw, "createdAt", "updatedAt")
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())
            ON CONFLICT DO NOTHING
        r   r   r   r   oddsjamr   z  Error storing prop: NF)cursorr   
isinstancestrr   r   rx   replaceexecuterg   dumpsr   nowr   utccommitrowcountrZ   r[   rollback)	connr   curr   r   r   	prop_typeraw_datar_   s	            r`   store_player_propr    s   +kkm9-fc"#''GF 9-wwvr"||~--c378$@	 
+wwy)1GGFO77=1wwy)
 	  GGM9-GGL!$GGFAJJx GGK >HLL$>

	  	||a &qc*+s   G G 	G4G//G4c                    	 | j                         }|j                  dd      }t        |t              r2t        j                  |j                         |j                               }|j                  d      xs# t        j                  t        j                        }|j                  dd       d|j                  dd       d|j                  d       }|j                  d      |j                  d	      |j                  d
      |j                  d      |j                  d      d}|j                  d||||j                  dd      |j                  dd      |j                  dd      |j                  dd      |j                  dd      |j                  dd      dt        j                  |      f       | j                          |j                   dkD  S # t"        $ r(}t%        d|        | j'                          Y d}~yd}~ww xY w)z6Store a game odds opportunity in GameOddsAltLine tabler   r	  r   r   r  r   z%Y%m%dr   r   r   r   r   )r   r   r   r   r   a  
            INSERT INTO "GameOddsAltLine"
            (league, "gameId", "gameDate", "homeTeam", "awayTeam", bookmaker, market, "lineValue", "oddsAmerican", source, raw)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON CONFLICT DO NOTHING
        r
  r   r   r   r   r   r  z  Error storing game odds: NF)r  r   r  r  r   r   rx   r   r  r   r  strftimer  rg   r  r  r  rZ   r[   r  )r  r   r  r   r   game_idr  r_   s           r`   store_game_oddsr!    s   )kkm9-fc"#''GFGGK(FHLL,F	WW[)45Qsww{I7V6WWXYbYkYkltYuXvw 
+wwy)177=1wwy)
 	  GGK$GGK$GGFI&GGHi(GGL!$GGFAJJx 
	$ 	||a +A3/0s   GG 	H HHc                 f   	 | j                         }|j                  dd      }|j                  d      xs# t        j                  t        j
                        }|j                  d|||j                  dd      |j                  dd      |j                  dd      |j                  d	d
      |j                  dd
      |j                  dd      |j                  dd
      |j                  dd      |j                  dd      t        j                  |      f       | j                          |j                  d
kD  S # t        j                  j                  $ r* | j                          t        |        t!        | |      cY S t"        $ r(}t%        d|        | j                          Y d}~yd}~ww xY w)z&Store a historical result with outcomer   r	  r   aH  
            INSERT INTO "OddsJamResult"
            (league, "gameDate", matchup, "playerName", market, "lineValue", "oddsAmerican",
             bookmaker, "edgePct", "sourceType", outcome, raw, "createdAt")
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())
            ON CONFLICT DO NOTHING
        r   r
  r   r   r   r   r   r   r   r   r   UNKNOWNz  Error storing result: NF)r  r   r   r  r   r  r  rg   r  r  r  psycopg2errorsUndefinedTabler  create_results_tablestore_historical_resultrZ   r[   )r  r   r  r   r   r_   s         r`   r(  r(    sb   %kkmHi0JJ{+Ix||HLL/I	 	  JJy"%JJ}b)JJx$JJ|Q'JJvq!JJvy)JJz1%JJ}i0JJy),JJv
	( 	||a??)) 5T"&tV44 (,-s   D7D: :AF0 F0F++F0c                     	 | j                         }|j                  d       | j                          t        d       y# t        $ r(}t        d|        | j                          Y d}~yd}~ww xY w)z.Create OddsJamResult table if it doesn't exista  
            CREATE TABLE IF NOT EXISTS "OddsJamResult" (
                id BIGSERIAL PRIMARY KEY,
                league VARCHAR(50) NOT NULL,
                "gameDate" TIMESTAMPTZ,
                matchup VARCHAR(255),
                "playerName" VARCHAR(255),
                market VARCHAR(100),
                "lineValue" FLOAT,
                "oddsAmerican" INT,
                bookmaker VARCHAR(100),
                "edgePct" FLOAT,
                "sourceType" VARCHAR(50),
                outcome VARCHAR(20),
                raw JSONB,
                "createdAt" TIMESTAMPTZ DEFAULT NOW(),
                UNIQUE(league, "gameDate", matchup, "playerName", market, "lineValue", bookmaker)
            );
            CREATE INDEX IF NOT EXISTS idx_oddsjam_result_league ON "OddsJamResult"(league);
            CREATE INDEX IF NOT EXISTS idx_oddsjam_result_date ON "OddsJamResult"("gameDate");
            CREATE INDEX IF NOT EXISTS idx_oddsjam_result_outcome ON "OddsJamResult"(outcome);
        z  Created OddsJamResult tablez  Error creating table: N)r  r  r  r[   rZ   r  )r  r  r_   s      r`   r'  r'  1  sb    kkm  	, 	-. (,-s   <? 	A0A++A0pagereturnc                 :   t        d       | j                  dd       t        j                  d       d| j                  v sd| j                  v rt        d       y	t        d
       	 | j                  dt               t        j                  d       t        j                         }t        d       	 | j                  d       t        j                  d       t        d       	 | j                  dd       t        d       t        |d      }|st        d       yt        d|        | j                  d      }|j                          t        j                  d       |D ]  }| j                  j                  |d        ! t        j                  d       | j                  j                  d!       t        j                  d"       t        d#       t        d$      D ]E  }| j                  }d%|j                         vrt        d&|         y	t        j                  d       G y#  | j                  dt               Y xY w#  | j                  d       Y xY w#  d| j                  v sd| j                  v rt        d       Y y	t        d       Y xY w)'zHandle OddsJam login with 2FAz&Step 1: Navigating to OddsJam login...zhttps://oddsjam.com/loginnetworkidle)
wait_untilrm   zbetting-tools	dashboardz  Already logged in!TzStep 2: Entering email...z#email-addresszinput[type="email"]rR   zStep 3: Clicking Sign In...zbutton.bg-brand-bluezbutton[type="submit"]r   z!Step 4: Waiting for 2FA screen...zinput[data-input-otp="true"]i'  r   z  Logged in without 2FA!z  2FA input not foundzStep 5: Retrieving 2FA code...   )r   r   z$  ERROR: Could not retrieve 2FA codeFzStep 6: Entering 2FA code: g      ?   )delayEnterr   z(Step 7: Waiting for login to complete...r   loginz  Login successful! URL: )r[   gotor   r   r^   fillODDSJAM_EMAILclickwait_for_selectorr   locatorkeyboardtypepressrangerx   )r*  login_trigger_timer   	otp_inputdigitr  current_urls          r`   login_to_oddsjamrD  R  s,   	
23II)mIDJJqM $(("kTXX&=$%	
%&8		"M2 	JJqM	
'(,

)* 	JJqM	
-.'=uM 

*++=sKD45	'v
./;<IOOJJsO -5,- 	JJqMMM JJqM	
452Y hh+++---k];<

1 g8		'7,

*+'dhh&+*A,-%&s*   (H* 3I %I! *II!)JJc                     d}t        d       	 | j                  dd       t        j                  d       t	        d      D ](  }| j                  d       t        j                  d	       * | j                  d
      }t        |d      }t        dt        |       d       |D ];  }d|j                  dd      v rt        ||      s$|dz  }*t        ||      s7|dz  }= 	 t        d       	 | j                  dd       t        j                  d       t	        d      D ](  }| j                  d       t        j                  d	       * | j                  d
      }t        |d      }	t        dt        |	       d       |	D ];  }d|j                  dd      v rt        ||      s$|dz  }*t        ||      s7|dz  }= 	 |S # t        $ r}t        d|        Y d}~d}~ww xY w# t        $ r}t        d|        Y d}~|S d}~ww xY w)z+Scrape live arbitrage and +EV opportunitiesr   zScraping Arbitrage page...z+https://oddsjam.com/betting-tools/arbitrage`  r0  r   r   .window.scrollTo(0, document.body.scrollHeight)rm   bodyr  r   z  Found z arbitrage opportunitiesPlayerr   r
  rR   z  Error scraping arbitrage: NzScraping Positive EV page...z-https://oddsjam.com/betting-tools/positive-evr  z positive EV opportunitiesz  Error scraping positive EV: )r[   r6  r   r   r?  evaluate
inner_textr   r   r   r  r!  rZ   )
r*  r  re   r  arb_textarb_oppsr   r_   ev_textev_oppss
             r`   scrape_live_opportunitiesrQ    s   M 

&'2		?	O

1 q 	AMMJKJJqM	 ??6*"8EX'?@A 	'C3778R00$T3/!Q&M"4-!Q&M	' 

()4		A5	Q

1 q 	AMMJKJJqM	 //&)!'}EW&@AB 	'C3778R00$T3/!Q&M"4-!Q&M	' 9  2,QC01122  4.qc2334sI   B9G 	G G 0B9G+ *G+ =G+ 	G(G##G(+	H4HHc                    d}t        d| d       g d}|D ]o  }t        d|        	 | j                  |d       t        j                  d       	 | j	                  d	      j
                  }|j                         r%|j                          t        j                  d
       	 | j	                  d      }|j                         r|j                          t        j                  d       | j	                  d      }|j
                  j                         r/|j
                  j                          t        j                  d       g }	d}
d}d}||k  rq| j                  d       t        j                  d       | j                  d      }t        |      }t        |      |
k(  r|dz  }nd}t        |      }
|}	|d
k\  rn||k  rqt        dt        |	       d       |	D ]  }|j                  d      rWt        j                  t        j                         t#        |      z
  }|d   j%                  t        j                         |k  rkt'        ||      r|dz  }d|j                  dd      v rt)        ||       t+        ||        r |S #  Y xY w#  Y UxY w# t,        $ r}t        d|        Y d}~d}~ww xY w)z:Scrape historical results from OddsJam bet tracker/resultsr   z"Scraping historical results (last z	 days)...)z-https://oddsjam.com/betting-tools/bet-trackerz)https://oddsjam.com/betting-tools/resultsz9https://oddsjam.com/betting-tools/positive-ev?tab=resultsz
  Trying: rF  r0  r   ztext=Resultsr   zB[data-testid="date-filter"], .date-filter, button:has-text("Date")rR   z*text=All Time, text=Last Year, text=Customrm   r   rG  rH  z
    Found z historical resultsr   )days)tzinforJ  r   r
  z    Error: N)r[   r6  r   r   r;  first
is_visibler9  rK  rL  r  r   r   r   r  r   r  r   r  r(  r  r!  rZ   )r*  r  	days_backre   results_urlsr^   results_tabdate_filterall_timeall_results
last_countscroll_attemptsmax_scrollsresults_textr  r   cutoffr_   s                     r`   scrape_historical_resultsrb    s   M	.yk
CDL  H
3% !F	IIc5I)JJqM"ll>:@@))+%%'JJqM
"ll+op))+%%'JJqM#||,XYH~~002 ,,.

1
 KJOK!K/NO

1#v6,\:w<:-#q(O&'O!$WJ")K"a' "K/" Js;/00CDE% 2::k*%\\(,,7):SSFk*22(,,2G&P *48!Q&M vzz(B77%dF3#D&12mHT }V  	Ks#$	sK   (JAJ%B JA=JCJJJJJ	J?&J::J?c                 j   d}|g d}|D ]  }t        d|j                          d       d| d| d| d	d| d
g}|D ]  }	 | j                  |d       t        j                  d       t        d      D ](  }| j                  d       t        j                  d       * | j                  d      }t        |d      }	|	D ]Q  }
|
j                  d      s||
d<   d|
j                  dd      v rt        ||
      s:|dz  }@t        ||
      sM|dz  }S   |S # t        $ r
}Y d}~d}~ww xY w)z*Scrape league-specific pages for more datar   N)r/   r2   r4   r5   r0   r3   z	Scraping z specific pages...z5https://oddsjam.com/betting-tools/positive-ev?league=z3https://oddsjam.com/betting-tools/arbitrage?league=zhttps://oddsjam.com/z/oddsz/player-propsi0u  r0  r   rm   rG  rR   rH  league_specificrI  r   rJ  r   r
  )r[   r   r6  r   r   r?  rK  rL  r   r   r  r!  rZ   )r*  r  leaguesre   r   urlsr^   r  r   oppsr   r_   s               r`   scrape_league_specificrh  )  sg   M@ #	&,,.));<= DF8LA&J"6(%0"6(-8	
  	C		#u	-

1 q "AMM"RSJJqM" v.&t9JK 	/C778,(.H3778R#88,T37)Q.M*45)Q.M	/	#J   s   B7D>DD	D2-D2c                 H   d}t               5 }|j                  j                  d      }|j                  dddd      }|j	                         }t        |      s%t        d	       |j                          	 d
d
d
       yt        j                  d       |dk(  s|dk(  r4t        d       |t        ||       z  }t        d       |t        ||       z  }|dk(  s
|dk(  s|dk(  rt        d       |t        || |      z  }|j                          d
d
d
       |S # 1 sw Y   |S xY w)zMain scraper functionr   T)headlessi  i8  )widthheightz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36)viewport
user_agentzERROR: Login failedNrm   dailyallz$
=== SCRAPING LIVE OPPORTUNITIES ===z'
=== SCRAPING LEAGUE-SPECIFIC PAGES ===backfillr  z$
=== SCRAPING HISTORICAL RESULTS ===)r   chromiumlaunchnew_contextnew_pagerD  r[   closer   r   rQ  rh  rb  )r  moderW  re   pbrowsercontextr*  s           r`   run_scraperr{  X  s/   M		 a**##T#2%%#t4U & 
 !  %'(MMO  	

17?dem9:6tTBBM <=3D$??M:$)2C9:6tT9MMM=@ A@ s   A*D BDD!c                     t        j                  d      } | j                  ddd       | j                  ddd       | j                  d	t        d
d       | j                  ddd       | j	                         }t        d       t        d       t        dt        j                  t        j                        j                                 t        d|j                  rdn|j                  rdnd        |j                  s|j                  rt        d|j                          t        d       t               }|st        d       y t               }	 t!        j"                  |      }t%        |       |j                  rd}n|j                  rd}nd}t'        |||j                        }|j)                          t        j                  t        j                        j                         |d<   |j+                  dd      |z   |d<   |j                  r4t        j                  t        j                        j                         |d<   t-        |       t        dd        t        d | d!       t        d"|d           t        d       y # t.        $ r,}t        d#|        dd l}|j3                          Y d }~y d }~ww xY w)$Nz%OddsJam Scraper with Backfill Support)descriptionz
--backfill
store_truez-Run in backfill mode to fetch historical data)actionhelpz--results-onlyzOnly scrape historical resultsz--daysm  zDays to backfill (default: 365))r=  defaultr  z	--leagues+zSpecific leagues to scrape)nargsr  z<============================================================zODDSJAM SCRAPER WITH AUTO-2FAzTime: zMode: rq  r  ro  zDays to backfill: zERROR: No database URL found)rw  rW  rc   re   r   rd   rw   z
COMPLETE: z records stored this runzTotal records all time: zERROR: )argparseArgumentParseradd_argumentr   
parse_argsr[   r   r  r   r  	isoformatrq  results_onlyrS  ra   ri   r$  connectr'  r{  rv  r   ru   rZ   	traceback	print_exc)	parserargsdb_urlrt   r  rw  totalr_   r  s	            r`   mainr    sP   $$1XYF
\@op
(Dde
sC>_`
35QRD	(O	
)*	F8<<-779:
;<	F:ARARIX_`
ab}}))"499+./	(O]F,-LE!' 	T" ==DDDDttyyA

 %LL6@@Bj!&?A!>!Fo==*2,,x||*D*N*N*PE&'58*o
5'!9:;()?(@ABh sms   D8J 	K"K  K__main__)r1  )live)r  r   )ro  r  )-__doc__ro   ry   sysr   globrg   r  r   r$  r   r   r   playwright.sync_apir   r   r   r8  r   rV   rf   r   r   r   ra   ri   ru   r}   r   r   r   r  r  r!  r(  r'  boolrD  r   rQ  rb  rh  r{  r  __name__ r~   r`   <module>r     sB  	 
 	 
       2 2 > >#$.H
 	8 h  
8  L	 
 	,  ,  K    	+  I  	)  y  K  [  y   
9! " # $ 
? F $"
R*D#LvrTn-`+\'TBA4 AD AH6D 63 6rWD W# Wt, ,c ,^$N8v zF r~   