#!/usr/bin/env python3
"""
Overnight Parameter Optimizer — TheCryptoClaw
Runs hundreds of parameter combinations:
  - Optimize on month 1 (e.g., Oct 2025)
  - Walk-forward validate on month 2 (e.g., Nov 2025)
  
Usage:
  nohup python3 overnight_optimizer.py > results/optimizer_log.txt 2>&1 &
  
  # Check progress:
  tail -f results/optimizer_log.txt
  cat results/optimizer_progress.json
"""

import os
import sys
import json
import time
import csv
import subprocess
import itertools
import signal
from datetime import datetime

os.chdir(os.path.dirname(os.path.abspath(__file__)))

# ===== CONFIGURATION =====

# Months: optimize on first, walk-forward on second
OPTIMIZE_MONTH = "202510"   # Oct 2025 — big active month
WALKFWD_MONTH  = "202511"   # Nov 2025 — different regime

# Starting balance for all tests
START_BALANCE = 20000.0

# Parameter grid
PARAM_GRID = {
    "LOT_SIZE":               [0.010, 0.015, 0.020, 0.025],
    "THREAD_PROFIT_TARGET":   [8, 10, 12, 15, 18, 20, 25],
    "MAX_INITIAL_ORDERS":     [8, 10, 12, 15],
    "MIN_ENTRY_SPACING":      [0, 25, 50, 100],
    "PROGRESSIVE_RLM_ENABLED":[False, True],
}

# Fixed settings (don't vary)
FIXED_SETTINGS = {
    "INITIAL_BALANCE":        START_BALANCE,
    "RECOVERY_PROFIT_TARGET": 30.0,
    "RECOVERY_LOT_MULTIPLIER":1.0,
    "ADAPTIVE_GRID":          True,
    "REFERENCE_ATR":          132.1,
    "COMMISSION_PERCENT":     0.025,  # Hyperliquid
    "PROGRESSIVE_RLM_START":  25,
    "PROGRESSIVE_RLM_RAMP":   True,
    "PROGRESSIVE_RLM_MAX":    1.3,
    "MAX_LEVERAGE":           10.0,
    "MARGIN_CAP_ENABLED":     False,
    "MARGIN_RESERVE_PCT":     0.20,
    "MARGIN_RELIEF_ENABLED":  False,
    "MARGIN_RELIEF_LEVEL":    4,
    "MARGIN_RELIEF_CLOSE_SUBS": True,
}

# Timeout per single-month backtest (seconds)
BACKTEST_TIMEOUT = 300

# Results files
RESULTS_DIR = "results"
PROGRESS_FILE = os.path.join(RESULTS_DIR, "optimizer_progress.json")
RESULTS_CSV = os.path.join(RESULTS_DIR, f"optimizer_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv")
SETTINGS_FILE = "settings.py"

# Original settings backup
SETTINGS_BACKUP = None

# ===== HELPER FUNCTIONS =====

def read_settings():
    """Read current settings.py as text."""
    with open(SETTINGS_FILE, 'r') as f:
        return f.read()

def write_setting(text, key, value):
    """Replace a setting value in settings.py text."""
    import re
    # Handle bool values
    if isinstance(value, bool):
        val_str = "True" if value else "False"
    elif isinstance(value, float):
        val_str = str(value)
    elif isinstance(value, int):
        val_str = str(value)
    else:
        val_str = str(value)
    
    # Try to replace existing setting
    pattern = rf'^{key}\s*=\s*.*$'
    new_line = f'{key} = {val_str}'
    new_text, count = re.subn(pattern, new_line, text, flags=re.MULTILINE)
    
    if count == 0:
        print(f"  WARNING: Setting {key} not found in settings.py!")
        return text
    return new_text

def apply_settings(params):
    """Apply a set of parameters to settings.py."""
    text = read_settings()
    
    # Apply fixed settings first
    for key, value in FIXED_SETTINGS.items():
        text = write_setting(text, key, value)
    
    # Apply variable params
    for key, value in params.items():
        text = write_setting(text, key, value)
    
    with open(SETTINGS_FILE, 'w') as f:
        f.write(text)

def run_backtest(month, balance, timeout=BACKTEST_TIMEOUT):
    """Run a single-month backtest and return results dict."""
    # Update balance
    text = read_settings()
    text = write_setting(text, "INITIAL_BALANCE", balance)
    with open(SETTINGS_FILE, 'w') as f:
        f.write(text)
    
    # Run bot_runner.py directly (not through bot_backtest.sh)
    cmd = [
        sys.executable, "bot_runner.py",
        f"--csv={month}",
        "--max-rows=0",
        "--tf=5min",
        "--throttle=0",
        f"--timeout={timeout}"
    ]
    
    try:
        result = subprocess.run(
            cmd, capture_output=True, text=True, timeout=timeout + 30,
            cwd=os.path.dirname(os.path.abspath(__file__))
        )
    except subprocess.TimeoutExpired:
        return {"error": "timeout", "net_profit": 0, "return_pct": 0, 
                "total_trades": 0, "max_dd_pct": 100, "profit_factor": 0}
    
    # Read results from the latest result JSON file
    try:
        import glob
        result_files = sorted(glob.glob(os.path.join(RESULTS_DIR, "result_*.json")), reverse=True)
        if not result_files:
            return {"error": "no result file", "net_profit": 0, "return_pct": 0,
                    "total_trades": 0, "max_dd_pct": 100, "profit_factor": 0}
        
        with open(result_files[0], 'r') as f:
            progress = json.load(f)
        
        if progress.get("status", "").lower() == "complete":
            final_balance = balance + progress.get("net_profit", 0)
            return {
                "net_profit": progress.get("net_profit", 0),
                "return_pct": progress.get("return_pct", 0),
                "total_trades": progress.get("total_trades", 0),
                "win_rate": progress.get("win_rate", 0),
                "profit_factor": progress.get("profit_factor", 0),
                "max_dd_pct": progress.get("max_dd_pct", 0),
                "gross_profit": progress.get("gross_profit", 0),
                "gross_loss": progress.get("gross_loss", 0),
                "final_balance": final_balance,
                "min_equity": progress.get("min_equity", balance),
                "max_equity": progress.get("max_equity", balance),
            }
        else:
            return {"error": f"status={progress.get('status','unknown')}", 
                    "net_profit": 0, "return_pct": 0, "total_trades": 0,
                    "max_dd_pct": 100, "profit_factor": 0}
    except Exception as e:
        return {"error": str(e), "net_profit": 0, "return_pct": 0, 
                "total_trades": 0, "max_dd_pct": 100, "profit_factor": 0}

def compute_rar(return_pct, max_dd_pct):
    """Risk-Adjusted Return = return% / max_dd%"""
    if max_dd_pct <= 0:
        return 0 if return_pct <= 0 else 999
    return round(return_pct / max_dd_pct, 2)

def generate_combos():
    """Generate all parameter combinations."""
    keys = sorted(PARAM_GRID.keys())
    values = [PARAM_GRID[k] for k in keys]
    combos = []
    for combo_vals in itertools.product(*values):
        combo = dict(zip(keys, combo_vals))
        combos.append(combo)
    return combos

def save_progress(data):
    """Save progress to JSON file."""
    with open(PROGRESS_FILE, 'w') as f:
        json.dump(data, f, indent=2)

# ===== MAIN =====

def main():
    global SETTINGS_BACKUP
    
    print("=" * 70)
    print("  OVERNIGHT PARAMETER OPTIMIZER — TheCryptoClaw")
    print(f"  Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"  Optimize month: {OPTIMIZE_MONTH}")
    print(f"  Walk-forward month: {WALKFWD_MONTH}")
    print(f"  Starting balance: ${START_BALANCE:,.0f}")
    print("=" * 70)
    
    # Backup original settings
    SETTINGS_BACKUP = read_settings()
    
    # Generate all combos
    combos = generate_combos()
    total = len(combos)
    print(f"\nTotal combinations: {total}")
    print(f"Estimated time: {total * 50 / 3600:.1f} hours ({total * 50}s)")
    print(f"Results: {RESULTS_CSV}")
    print()
    
    # Setup CSV results file
    csv_fields = [
        "combo_id", "lot_size", "thread_tp", "max_orders", "entry_spacing", "prog_rlm",
        # Optimize month
        "opt_net", "opt_return_pct", "opt_trades", "opt_wr", "opt_pf", "opt_dd", "opt_rar",
        "opt_gross_profit", "opt_gross_loss",
        # Walk-forward month  
        "wf_net", "wf_return_pct", "wf_trades", "wf_wr", "wf_pf", "wf_dd", "wf_rar",
        "wf_gross_profit", "wf_gross_loss",
        # Combined scores
        "combined_return", "combined_dd", "combined_rar", "combined_pf",
        "opt_final_balance", "wf_final_balance",
        "error"
    ]
    
    with open(RESULTS_CSV, 'w', newline='') as f:
        writer = csv.DictWriter(f, fieldnames=csv_fields)
        writer.writeheader()
    
    # Run all combinations
    results = []
    start_time = time.time()
    
    for i, combo in enumerate(combos):
        combo_id = i + 1
        elapsed = time.time() - start_time
        avg_time = elapsed / max(1, i) 
        eta = avg_time * (total - i)
        
        print(f"\n[{combo_id}/{total}] ETA: {eta/60:.0f}min | "
              f"LOT={combo['LOT_SIZE']} TTP={combo['THREAD_PROFIT_TARGET']} "
              f"MO={combo['MAX_INITIAL_ORDERS']} SP={combo['MIN_ENTRY_SPACING']} "
              f"RLM={'Y' if combo['PROGRESSIVE_RLM_ENABLED'] else 'N'}")
        
        row = {
            "combo_id": combo_id,
            "lot_size": combo["LOT_SIZE"],
            "thread_tp": combo["THREAD_PROFIT_TARGET"],
            "max_orders": combo["MAX_INITIAL_ORDERS"],
            "entry_spacing": combo["MIN_ENTRY_SPACING"],
            "prog_rlm": combo["PROGRESSIVE_RLM_ENABLED"],
            "error": ""
        }
        
        try:
            # Apply settings
            apply_settings(combo)
            
            # ---- OPTIMIZE MONTH ----
            t0 = time.time()
            opt = run_backtest(OPTIMIZE_MONTH, START_BALANCE)
            t1 = time.time()
            
            if "error" in opt:
                row["error"] = f"opt: {opt['error']}"
                print(f"  OPT ERROR: {opt['error']}")
            else:
                opt_rar = compute_rar(opt["return_pct"], opt["max_dd_pct"])
                row.update({
                    "opt_net": opt["net_profit"],
                    "opt_return_pct": opt["return_pct"],
                    "opt_trades": opt["total_trades"],
                    "opt_wr": opt.get("win_rate", 0),
                    "opt_pf": opt["profit_factor"],
                    "opt_dd": opt["max_dd_pct"],
                    "opt_rar": opt_rar,
                    "opt_gross_profit": opt.get("gross_profit", 0),
                    "opt_gross_loss": opt.get("gross_loss", 0),
                    "opt_final_balance": opt.get("final_balance", START_BALANCE),
                })
                print(f"  OPT: ${opt['net_profit']:+,.0f} ({opt['return_pct']:+.1f}%) DD:{opt['max_dd_pct']:.1f}% "
                      f"PF:{opt['profit_factor']:.2f} RAR:{opt_rar} [{t1-t0:.0f}s]")
                
                # ---- WALK-FORWARD MONTH ----
                # Use the ending balance from optimize as starting balance
                wf_balance = opt.get("final_balance", START_BALANCE + opt["net_profit"])
                t2 = time.time()
                wf = run_backtest(WALKFWD_MONTH, wf_balance)
                t3 = time.time()
                
                if "error" in wf:
                    row["error"] = f"wf: {wf['error']}"
                    print(f"  WF ERROR: {wf['error']}")
                else:
                    wf_rar = compute_rar(wf["return_pct"], wf["max_dd_pct"])
                    row.update({
                        "wf_net": wf["net_profit"],
                        "wf_return_pct": wf["return_pct"],
                        "wf_trades": wf["total_trades"],
                        "wf_wr": wf.get("win_rate", 0),
                        "wf_pf": wf["profit_factor"],
                        "wf_dd": wf["max_dd_pct"],
                        "wf_rar": wf_rar,
                        "wf_gross_profit": wf.get("gross_profit", 0),
                        "wf_gross_loss": wf.get("gross_loss", 0),
                        "wf_final_balance": wf.get("final_balance", wf_balance),
                    })
                    
                    # Combined scores
                    combined_return = ((1 + opt["return_pct"]/100) * (1 + wf["return_pct"]/100) - 1) * 100
                    combined_dd = max(opt["max_dd_pct"], wf["max_dd_pct"])
                    combined_rar = compute_rar(combined_return, combined_dd)
                    combined_gp = opt.get("gross_profit", 0) + wf.get("gross_profit", 0)
                    combined_gl = opt.get("gross_loss", 0) + wf.get("gross_loss", 0)
                    combined_pf = round(combined_gp / combined_gl, 2) if combined_gl > 0 else 999
                    
                    row.update({
                        "combined_return": round(combined_return, 2),
                        "combined_dd": combined_dd,
                        "combined_rar": combined_rar,
                        "combined_pf": combined_pf,
                    })
                    
                    print(f"  WF:  ${wf['net_profit']:+,.0f} ({wf['return_pct']:+.1f}%) DD:{wf['max_dd_pct']:.1f}% "
                          f"PF:{wf['profit_factor']:.2f} RAR:{wf_rar} [{t3-t2:.0f}s]")
                    print(f"  COMBINED: {combined_return:+.1f}% DD:{combined_dd:.1f}% RAR:{combined_rar} PF:{combined_pf}")
        
        except Exception as e:
            row["error"] = str(e)
            print(f"  EXCEPTION: {e}")
        
        # Append row to CSV
        with open(RESULTS_CSV, 'a', newline='') as f:
            writer = csv.DictWriter(f, fieldnames=csv_fields)
            writer.writerow(row)
        
        results.append(row)
        
        # Update progress
        save_progress({
            "status": "RUNNING",
            "current": combo_id,
            "total": total,
            "pct": round(combo_id / total * 100, 1),
            "elapsed_sec": round(time.time() - start_time),
            "eta_sec": round(eta),
            "results_file": RESULTS_CSV,
            "last_combo": {k: str(v) for k, v in combo.items()},
            "timestamp": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        })
    
    # ===== COMPLETE =====
    total_time = time.time() - start_time
    
    # Restore original settings
    with open(SETTINGS_FILE, 'w') as f:
        f.write(SETTINGS_BACKUP)
    print("\n✅ Original settings.py restored")
    
    # Sort by combined RAR
    valid = [r for r in results if r.get("combined_rar") and r.get("combined_rar") > 0]
    valid.sort(key=lambda x: x.get("combined_rar", 0), reverse=True)
    
    print("\n" + "=" * 80)
    print(f"  OPTIMIZATION COMPLETE — {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"  Total time: {total_time/3600:.1f} hours ({total_time:.0f}s)")
    print(f"  Combinations tested: {total}")
    print(f"  Valid results: {len(valid)}")
    print(f"  Results saved: {RESULTS_CSV}")
    print("=" * 80)
    
    # Top 20 by combined RAR
    print(f"\n{'='*100}")
    print(f"  TOP 20 BY COMBINED RAR (Optimize: {OPTIMIZE_MONTH} → Walk-Forward: {WALKFWD_MONTH})")
    print(f"{'='*100}")
    print(f"{'#':>3} {'Lot':>5} {'TTP':>4} {'MO':>3} {'SP':>4} {'RLM':>4} | "
          f"{'Opt%':>7} {'OptDD':>6} {'OptPF':>6} | "
          f"{'WF%':>7} {'WFDD':>6} {'WFPF':>6} | "
          f"{'Comb%':>8} {'CDD':>5} {'CRAR':>6} {'CPF':>5}")
    print("-" * 100)
    
    for rank, r in enumerate(valid[:20], 1):
        print(f"{rank:>3} {r['lot_size']:>5} {r['thread_tp']:>4} {r['max_orders']:>3} "
              f"{r['entry_spacing']:>4} {'Y' if r['prog_rlm'] else 'N':>4} | "
              f"{r.get('opt_return_pct',0):>+6.1f}% {r.get('opt_dd',0):>5.1f}% {r.get('opt_pf',0):>5.2f} | "
              f"{r.get('wf_return_pct',0):>+6.1f}% {r.get('wf_dd',0):>5.1f}% {r.get('wf_pf',0):>5.2f} | "
              f"{r.get('combined_return',0):>+7.1f}% {r.get('combined_dd',0):>4.1f}% {r.get('combined_rar',0):>5.1f} {r.get('combined_pf',0):>5.2f}")
    
    # Top 20 by combined return
    valid2 = sorted(valid, key=lambda x: x.get("combined_return", 0), reverse=True)
    print(f"\n{'='*100}")
    print(f"  TOP 20 BY COMBINED RETURN %")
    print(f"{'='*100}")
    print(f"{'#':>3} {'Lot':>5} {'TTP':>4} {'MO':>3} {'SP':>4} {'RLM':>4} | "
          f"{'Opt%':>7} {'OptDD':>6} {'OptPF':>6} | "
          f"{'WF%':>7} {'WFDD':>6} {'WFPF':>6} | "
          f"{'Comb%':>8} {'CDD':>5} {'CRAR':>6} {'CPF':>5}")
    print("-" * 100)
    
    for rank, r in enumerate(valid2[:20], 1):
        print(f"{rank:>3} {r['lot_size']:>5} {r['thread_tp']:>4} {r['max_orders']:>3} "
              f"{r['entry_spacing']:>4} {'Y' if r['prog_rlm'] else 'N':>4} | "
              f"{r.get('opt_return_pct',0):>+6.1f}% {r.get('opt_dd',0):>5.1f}% {r.get('opt_pf',0):>5.2f} | "
              f"{r.get('wf_return_pct',0):>+6.1f}% {r.get('wf_dd',0):>5.1f}% {r.get('wf_pf',0):>5.2f} | "
              f"{r.get('combined_return',0):>+7.1f}% {r.get('combined_dd',0):>4.1f}% {r.get('combined_rar',0):>5.1f} {r.get('combined_pf',0):>5.2f}")
    
    # Top 10 lowest DD with positive returns
    valid3 = [r for r in valid if r.get("combined_return", 0) > 0]
    valid3.sort(key=lambda x: x.get("combined_dd", 100))
    print(f"\n{'='*100}")
    print(f"  TOP 10 LOWEST DRAWDOWN (positive returns only)")
    print(f"{'='*100}")
    for rank, r in enumerate(valid3[:10], 1):
        print(f"{rank:>3} {r['lot_size']:>5} {r['thread_tp']:>4} {r['max_orders']:>3} "
              f"{r['entry_spacing']:>4} {'Y' if r['prog_rlm'] else 'N':>4} | "
              f"Comb: {r.get('combined_return',0):>+7.1f}% DD: {r.get('combined_dd',0):>4.1f}% "
              f"RAR: {r.get('combined_rar',0):>5.1f} PF: {r.get('combined_pf',0):>5.2f}")
    
    # Save final summary
    save_progress({
        "status": "COMPLETE",
        "total": total,
        "valid": len(valid),
        "total_time_sec": round(total_time),
        "results_file": RESULTS_CSV,
        "top1_rar": valid[0] if valid else None,
        "top1_return": valid2[0] if valid2 else None,
        "top1_low_dd": valid3[0] if valid3 else None,
        "timestamp": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
    })
    
    print(f"\n✅ All done! Results in {RESULTS_CSV}")
    print(f"   Progress/summary in {PROGRESS_FILE}")

if __name__ == "__main__":
    # Handle graceful shutdown
    def cleanup(signum, frame):
        print(f"\n⚠️ Signal {signum} received, restoring settings and exiting...")
        if SETTINGS_BACKUP:
            with open(SETTINGS_FILE, 'w') as f:
                f.write(SETTINGS_BACKUP)
            print("✅ Settings restored")
        save_progress({"status": "KILLED", "timestamp": datetime.now().strftime('%Y-%m-%d %H:%M:%S')})
        sys.exit(1)
    
    signal.signal(signal.SIGTERM, cleanup)
    signal.signal(signal.SIGINT, cleanup)
    
    main()