
    7iN                        d Z ddlZddlZddlZddlZddlZddlZddlZddlmZ  ej                  ej                  j                  ej                  j                  e                   ej                  j                  dd       	  ej                  d       ej                  j#                  ej                  j                  ej                  j                  e            d      Zej                  j#                  ej                  j                  ej                  j                  e            d      Zd Zd	 Zd
efdZi a ej2                         Zd
efdZd ZdedefdZ G d d      Z d Z!d Z"e#dk(  r e!        yy# e$ r Y w xY w)a*  
Threaded backtest runner for TheCryptoClaw bot.

Features:
- Worker thread for backtest execution (non-blocking)
- Progress file updated every 500 candles (bot can poll)
- CPU throttling via os.nice() and periodic sleeps
- Hard timeout via signal alarm
- Chunked CSV loading for memory efficiency
    N)datetime.
   zbacktest_progress.jsonresultsc                     t         dz   }	 t        |d      5 }t        j                  | |dt               ddd       t        j                  |t                y# 1 sw Y   $xY w# t        $ r Y yw xY w)zAtomically write progress JSON.z.tmpw   )indentdefaultN)PROGRESS_FILEopenjsondumpstrosreplace	Exception)datatmpfs      ,/var/www/html/crpytotradingbot/bot_runner.pywrite_progressr   "   sd    
&
 C#s^ 	6qIIdAa5	6


3&	6 	6  s'   A$ A"A$ A!A$ $	A0/A0c                     t        ddt        j                         j                         d       t	        d       t        j                  d       y)z+Hard timeout - write final status and exit.timeoutzHard timeout reachedstatusmessage
updated_atz
HARD TIMEOUT - backtest killed   N)r   r   now	isoformatprintr   _exit)signumframes     r   timeout_handlerr&   ,   s9    i4JZbZfZfZhZrZrZtuv	
,-HHQK    r   c                    t        j                  t        d       t        j                         j                  d      }t         j                  j                  t        d| d      }	 t        |d      5 }t        j                  | |d       d	d	d	       t        d
|        t        |        y	# 1 sw Y   #xY w# t        $ r}t        d|        Y d	}~4d	}~ww xY w)zSave final result to both progress file and timestamped results JSON.
    
    Called on completion AND on graceful shutdown (SIGTERM/SIGINT) so results
    are never lost even if the process is killed.
    Texist_ok%Y%m%d_%H%M%Sresult_z.jsonr   r	   )r
   NzResults saved to z!WARNING: Failed to save results: )r   makedirsRESULTS_DIRr   r    strftimepathjoinr   r   r   r"   r   r   )r   run_idresult_pathr   es        r   _save_final_resultr5   3   s     KKd+\\^$$_5F'',,{gfXU,CDK7+s# 	)qIIdAa(	)!+/0 4	) 	)  71!5667s0   (B; 4B/B; /B84B; ;	CCCc                 Z    t         5  | j                         addd       y# 1 sw Y   yxY w)z=Thread-safe update of last known progress for crash recovery.N)_last_progress_lockcopy_last_progress)r   s    r   _update_last_progressr:   J   s'     
 %% % %s   !*c                 P   t         5  t        j                         }ddd       r`|j                  d      dk(  rLd|d<   d|  |d<   t	        j
                         j                         |d<   t        |       t        d|  d	       t        j                  d
       y# 1 sw Y   xY w)z7Handle SIGTERM/SIGINT - save whatever progress we have.Nr   runninginterruptedzProcess killed by signal r   r   z
Interrupted by signal z - progress savedr   )r7   r9   r8   getr   r    r!   r5   r"   r   r#   )r$   r%   progresss      r   _graceful_shutdownr@   P   s    	 )!&&()HLL*i7* 9&B!)!9!9!;8$(0ABCHHQK) )s   BB%csv_argreturnc                 F   t         j                  j                  t         j                  j                  t         j                  j	                  t
                    dd      }t         j                  j                  |d      }t         j                  j                  |       r| gS | dv rt         j                  j                  |      rmt        |      5 }t        j                  |      }ddd       t        j                               }t        |      dk\  r|dd n|}|D cg c]
  }||   d    c}S d	gS | d
v rxt         j                  j                  |      rVt        |      5 }t        j                  |      }ddd       t        j                               D cg c]
  }||   d    c}S dgS t         j                  j                  |      st        d| d       d	gS t        |      5 }t        j                  |      }ddd       t        j                               }t        |       dk(  r+| j                         r| |v r	||    d   gS t        d|        d| v rXt        |       dk(  rJ| j!                  d      \  }}	|D cg c]  }||cxk  r|	k  sn n||   d    }
}|
st        d|        |
S ddl}|j%                  d|       }|rGt'        |j)                  d            }|t        |      k\  rd	gS || d }|D cg c]
  }||   d    c}S t         j                  j                  ||       }t         j                  j                  |      r|gS t        d|        # 1 sw Y   xY wc c}w # 1 sw Y   xY wc c}w # 1 sw Y   xY wc c}w c c}w )u  
    Resolve CSV argument to one or more file paths.
    
    Supports:
      - Direct file path: "BTCUSD_PAST6MONTHS.csv" or "data/monthly/BTCUSD_202601.csv"
      - Month shorthand: "1month" → latest 1 month, "3month" → latest 3 months
      - Specific month: "202601" → data/monthly/BTCUSD_202601.csv
      - Month range: "202510-202512" → Oct, Nov, Dec 2025
      - Legacy: "6month" → full 6-month file, "5year" → full 5-year file
    
    Returns list of CSV file paths to load in order.
    r   monthlyz
index.json)6monthBTCUSD_PAST6MONTHS.csvN   ir0   rF   )5year
BTCUSD.csvrI   z$WARNING: Monthly index not found at z, falling back to full file   zNo monthly data for -   zNo data in range r   z^(\d+)month$r   zCannot resolve CSV: )r   r0   r1   dirnameabspath__file__isfiler   r   loadsortedkeyslenr"   isdigitFileNotFoundErrorsplitrematchintgroup)rA   MONTHLY_DIR
INDEX_FILEr   index	availableselectedmstart_mend_mpathsrX   rY   nmonthly_paths                  r   resolve_csv_pathsrg   \   s8    '',,rwwrwwx/HI6S\]Kk<8J 
ww~~gy 6677>>*%j! %Q		!%uzz|,I),Y1)<y~)H.67E!HV$77())))77>>*%j! %Q		!%.4UZZ\.BCE!HV$CC~ 77>>*%4ZL@[\]())	j	 Q		! uzz|$I 7|qW__.e'N6*++"6wi @AA g~#g,", s++4Na18M8Mq&!NN#&7y$ABB HH_g.EAI,--aRS>*23Qa 33 77<<W5L	ww~~l#~
27)<
==o% %
 8
% %C  O 4sB   M( M5#M:N-NNNN(M2:NNc                   *    e Zd ZdZddZd Zd Zd Zy)ThrottledBacktestz4Run backtest with throttling and progress reporting.c                 f    || _         || _        || _        || _        d | _        d | _        d | _        y N)csv_pathmax_rows	timeframethrottle_msresulterror
start_time)selfrl   rm   rn   ro   s        r   __init__zThrottledBacktest.__init__   s4      "&
r'   c                 v   ddl }ddlm}m} t	        | j
                        }dj                  d |D              }t        dd| d| j                   d	|D cg c]!  }t        j                  j                  |      # c}t        j                         j                         d
       g }d}| j                  r| j                  dkD  r| j                  nd}	|D ]  }
d|
i}|	
|	dkD  r|	|d<    |j                  d3i |}|j!                  |       |t#        |      z  }|	|	t#        |      z  }	|	dk  r nbt        ddt        j                  j                  |
       dt#        |      dd|dt        j                         j                         d        t#        |      dkD  r|j%                  |d      n|d   }~t#        |      }t        dd|dd| j&                   d|t        j                         j                         d       |j)                  |d   j*                  j-                  ddd      d      |d <   |d!   |d"   z   d#z  |d$<   |j/                  d       }|d$   j1                  | j&                        j3                         }g d%|_        |d&   j1                  | j&                        j7                         |d'<   |j9                         }~ddl}|j=                          ddl} ||j@                        }g }|jC                         D ]l  \  }}|j!                   ||jE                         tG        |d(         tG        |d)         tG        |d*         tG        |d+         tG        |d'         ,             n ||_$        t        d-d.t#        |      dd/|dd0|t#        |      t+        |jJ                  d         t+        |jJ                  d1         tG        |d*   jM                               tG        |d)   jO                               t        j                         j                         d2	       |S c c}w )4zDLoad tick data and resample to candles. Supports multi-file loading.r   N)
DataLoaderOHLCVz, c              3   Z   K   | ]#  }t         j                  j                  |       % y wrk   )r   r0   basename).0r   s     r   	<genexpr>z.ThrottledBacktest.load_data.<locals>.<genexpr>   s     Eabgg..q1Es   )+loadingzLoading z (max z	 rows)...)r   r   filesr   filepath_or_buffernrowszLoaded  (,z ticks). Total: r   r   T)ignore_index
resamplingzResampling z
 ticks to z candles...)r   r   ticks_loadedr   	Timestampz	:(\d{3})$z.\1)regexz%Y%m%d %H:%M:%S.%f)formatr   z	Bid pricez	Ask priceg       @mid)r   highlowclosez
Bid volumevolumer   r   r   r   )	timestampr   r   r   r   r   readyzReady: z candles from z ticks)	r   r   r   candles
date_startdate_end	price_low
price_highr    )(pandasdata_loaderrv   rw   rg   rl   r1   r   rm   r   r0   ry   r   r    r!   read_csvappendrT   concatrn   to_datetimer   r   	set_indexresampleohlccolumnssumdropnagccollectsettingsSYMBOLiterrowsto_pydatetimefloatr   r^   minmax)rs   pdrv   rw   	csv_files	file_descr   framestotal_loaded	rows_leftcsv_fileread_paramschunkdf
tick_countohlcvr   r   loaderr   tsrows                         r   	load_datazThrottledBacktest.load_data   s   1 &dmm4	IIE9EE	!)F4==/K3<=abgg&&q)=",,.224	
 	 &*mm8IDMMPT	! 	H/:K$Q'0G$BKK.+.EMM% CJ&L$SZ'	>#$RWW%5%5h%?$@3u:a.P`amno`pq&lln668 	( 69[1_RYYvDY1&QR)W
"$ZN*T^^<LKX&",,.224	
 	 {O''fD'I' ( 
: _r+6#=5	\\*%5	""4>>27798\*33DNNCGGIh 


 	HOO,~~' 	GBNN5**,3v;'3v;'#e*%CL)S]+ 	  Wa 0z!nFS&7|ekk!n-EKKO,uU|//12f 1 1 34",,.224

 
	 k >s   &P6c                    ddl }|j                  j                  }|syt        j                  t
        d       t        j                         j                  d      }t        j                  j                  t
        d| d      }t        |dd	
      5 }|j                  |      }|j                  g d       |D ]  }d}	|j                  r@|j                  r4t!        |j                  |j                  z
  j#                         dz  d      }	d}
|j$                  rZ|j$                  dkD  rK|j&                  dkD  r<|j&                  |j$                  z  }|dkD  rt!        |j(                  |z  dz  d      nd}
|j                  |j*                  t-        |j.                  d      r|j.                  j0                  nt3        |j.                        t!        |j&                  d      |j$                  rt!        |j$                  d      nd|j                  rt3        |j                        nd|j4                  rt!        |j4                  d      nd|j                  rt3        |j                        ndt!        |j(                  d      |j6                  |j8                  |	|
g        	 ddd       t;        d| dt=        |       d       i }|D ]  }|j6                  }||vr	dddd||<   ||   dxx   dz  cc<   ||   dxx   |j(                  z  cc<   |j8                  sSd|j8                  v sb	 t?        |j8                  jA                  d      d   jA                  d      d         }tC        ||   d   |      ||   d<    t        j                  j                  t
        d| d      }t        |dd	
      5 }|j                  |      }|j                  g d       tI        |jK                               D ]-  \  }}|j                  ||d   t!        |d   d      |d   g       / 	 ddd       t;        d| dt=        |       d       y# 1 sw Y   xY w# tD        tF        f$ r Y w xY w# 1 sw Y   HxY w)a'  Export full trade-by-trade log to CSV for post-run analysis.
        
        Columns: trade_id, side, lots, entry_price, entry_time, exit_price,
                 exit_time, profit, commission, magic (thread_id), comment 
                 (recovery level info), duration_sec, return_pct
        r   NTr)   r+   trades_z.csvr    )newline)trade_idsidelotsentry_price
entry_time
exit_price	exit_timeprofitmagic_threadcommentduration_min
return_pct<   r   d      valuer	   zTrade log exported: r   z trades))tradesr   	max_levelr   r   REC_L_r   threads_)	thread_idr   total_profitmax_recovery_levelzThread summary: z	 threads))&csvorder_enginetrade_historyr   r-   r.   r   r    r/   r0   r1   r   writerwriterow	open_time
close_timeroundtotal_secondspricer   r   order_idhasattrr   r   r   close_pricemagicr   r"   rT   rZ   rW   r   
ValueError
IndexErrorrR   items)rs   enginer   r   r2   log_pathr   r   tr   r   
cost_basisthread_statsr   levelthread_pathstatss                    r   _export_trade_logz#ThrottledBacktest._export_trade_log  s    	$$22
K$/((977<<wvhd-CD(C, !	ZZ]FOO    #;;1<<#(!,,*D)S)S)UXZ)Z\]#^L "
77qww{qvvz!"!''!1JLVYZN:(='Da!H`aJJJ$+AFFG$<AFFLL#aff+!&&!$)*E!''1%d()C$/0}}E!--+$)*C%4!((A&GGII ! !	F 	$XJbVXFG  	A''C,&/0AA$NS!h'1,'h'1883'yyW		1		 8 ; A A# Fq IJE58c9J;9WY^5_L%k2	 ggll;(6($0GH+sB/ 	g1ZZ]FOOYZ$\%7%7%9: g
UeHouU8_a7PRWXcRd efg	g 	 RL0A/B)LMy!	 !	d #J/ 	g 	gs-   HQ)AQ2A.Q(QQ%$Q%(Q1c                 d   ddl }ddlm} ddlm}  |dd       t        j
                         | _        | j                         }t        |      } ||j                  |j                  |j                  |j                        }d}|j                         D ]d  }|j                  |       |d	z  }| j                  dkD  r*|d
z  dk(  r"t        j                   | j                  dz         |dz  dk(  s||k(  sat        j
                         | j                  z
  }	|j"                  j%                         }
|j"                  j&                  }||z  d
z  }|	dkD  r||	z  nd}|dkD  r||z
  |z  nd}i ddd|d|dt)        |d	      dt)        |d      dt)        |
d      dt)        ||j                  z
  |j                  z  d
z  d      dt)        |j*                  d      dt)        |j,                  d      dt)        |j.                  dkD  r|j0                  |j.                  z  d
z  ndd      dt        |j"                  j2                        dt        |j"                  j4                        dt)        |j6                  d      dt9        |j:                        dt)        |	d	      dt)        |d	      dt)        |d	      d t=        j>                         jA                         i}tC        |       tE        |       g dd!lm#} |j"                  jI                         \  }}tK        |j"                  j4                  jM                               D ]h  }|jN                  |jP                  k(  r'|j"                  jS                  |jT                  |       C|j"                  jS                  |jT                  |       j | jW                  |       |jY                  |      }t        j
                         | j                  z
  }	i dd"dt)        |	d	      d#t)        |jZ                  d      dt)        |j\                  d      d$|j^                  d%t)        |j`                  d      d&|jb                  dkD  r#t)        |jd                  |jb                  z  d      nd'dt)        |jf                  d      dt)        |j,                  d      dt)        |j*                  d      d(t)        |jd                  d      d)t)        |jb                  d      d*t)        |jh                  d      d+|jj                  d,|jl                  d-|jn                  rt9        |jn                        ndd.|jp                  rt9        |jp                        nd|t=        j>                         jA                         d/}ts        |       |S )0z6Run the backtest with throttling and progress updates.r   N)BacktestEngine)setup_loggingFT)consolefile)symbolinitial_balancecommission_percentslippage_pipsr   r   g     @@i  r   r<   	processedtotalpctbalancer	   equityr   
max_equity
min_equity
max_dd_pctr   open_positionscurrent_pricecurrent_dateelapsed_seccandles_per_seceta_secr   )	OrderSidecomplete
net_profittotal_tradeswin_rateprofit_factori  gross_profit
gross_loss
commissiontotal_threadsr   r   r   )r   r   ):r   backtestr   utilsr   timerr   r   rT   r   INITIAL_BALANCECOMMISSION_PERCENTSLIPPAGE_PIPSiterate_candlesprocess_candlero   sleepr   
get_equityr   r   r   r   peak_equitymax_drawdownr   	positionsr   r   r   r   r    r!   r   r:   r  get_current_pricelistvaluesr   BUYclose_positionposition_idr   _compile_resultstotal_returntotal_return_percentr  r  r  r  max_drawdown_percenttotal_commissionr  r   
start_dateend_dater5   )rs   r   r   r   r   r   r   r   candleelapsedr   r   pct_donerateetaprogress_datar  bidaskpositionrp   finals                         r   runzThrottledBacktest.runl  s   +'e$/))+ ~~D	  ??$44'::"00	
 	**, &	5F!!&)NI !#	C1(<

4++f45 3!#yE'9))+7,,779 --55$u,s2.5ky7*q481Huy(D0!!i!! U! 51-	!
 uWa0! eFA.! !%83K3K)KxOgOg(gjm(mop"q! !%(9(91"=! !%(9(91"=! !%\b\n\nqr\r)<)<v?Q?Q)QTW)Wxy{|"}! c&"5"5"C"CD! %c&*=*=*G*G&H! $U6<<%;! #C(8(8$9! "5!#4!  &uT1~!!" uS!}#!$ !(,,.":":"<%!( }-%m4M&	5R 	+&&88:SV00::AACD 	NH}}	-##2283G3GM##2283G3GM		N 	v& ((.))+/
j
5!,
 % 3 3Q7
 % ; ;Q?	

 F//
 fooq1
 RXRcRcfgRgU6#6#69J9J#JANmp
 % ; ;Q?
 % 1 115
 % 1 115
 E&"5"5q9
 % 1 115
 % 7 7;
 V11
 !&";";
  F4E4E#f//04!
" FOO,T#
$ ",,.224'
* 	5!r'   N)5minr	   )__name__
__module____qualname____doc__rt   r   r   r6  r   r'   r   ri   ri      s    >aFMN^gr'   ri   c                     t        j                  d      } | j                  ddd       | j                  dt        dd	
       | j                  ddd       | j                  dt        dd
       | j                  dt        dd
       | j                  ddd       | j	                         }t        j
                  t
        j                  t               t        j                  |j                         t        j
                  t
        j                  t               t        j
                  t
        j                  t               |j                  dkD  r|j                  nd }t        |j                  ||j                   |j"                        }|j$                  rPt'        j(                  t*        |fd      }|j-                          t/        dt0                |j3                          y |j5                         }|j7                          y )NzThrottled backtest runner)descriptionz--csv1monthz8CSV file or period (1month, 3month, 6month, 202601, etc))r   helpz
--max-rowsi zMax tick rows (0=all))typer   r?  z--tfr7  zCandle timeframez
--throttler	   zSleep ms per 100 candlesz	--timeouti  zHard timeout secondsz--background
store_truezRun in background thread)actionr?  r   )rl   rm   rn   ro   T)targetargsdaemonz4Backtest started in background thread. Monitor: cat )argparseArgumentParseradd_argumentrZ   
parse_argssignalSIGALRMr&   alarmr   SIGTERMr@   SIGINTrm   ri   r   tfthrottle
background	threadingThread_run_workerstartr"   r   r1   r6  print_summary)parserrD  rm   runnerthreadrp   s         r   mainrZ    s   $$1LMF
8rs
3F]^
5GH
3@Z[
#sAWX
|B\]D MM&../2
LL MM&.."45
MM&--!34 $ 1t}}tH''MM	F !!F9TRD]OTUr'   c                    	 | j                         }t        d       |j                          y# t        $ rP}t	        dt        |      t        j                         j                         d       t        d|        Y d}~yd}~ww xY w)z'Worker function for threaded execution.z=
============================================================rq   )r   rq   r   zERROR: N)	r6  r"   rV  r   r   r   r   r    r!   )rX  rp   r4   s      r   rT  rT    si    o 'CF(,,.JbJbJdefsms   +. 	BABB__main__)$r;  sysr   r   r  rJ  rR  rF  r   chdirr0   rM   rN   rO   insertniceOSErrorr1   r   r.   r   r&   dictr5   r9   Lockr7   r:   r@   r   r   rg   ri   rZ  rT  r8  r   r'   r   <module>rd     sd  	  	       2	3 4 3 	BGGBK RWW__RWW__X-FGIabggll277??277??8+DEyQT ( $inn& % %
O>s O>t O>de eP	$N zF Y  		s   F F
	F
