#!/usr/bin/env python3
"""Deterministic scanner-candidate reviewer for XAUUSD hybrid workflow.

This is NOT a final backtest engine. It mechanically triages scanner candidates
using strict chronological OHLCV rules, writes review labels, and creates a manual
replay queue. Only candidates that pass the mechanical filter become
`needs_manual_replay`; nothing is promoted to confirmed journal trades here.

No raw OHLCV is printed; stdout is summary statistics only.
"""
from __future__ import annotations

import argparse
import json
import statistics
from bisect import bisect_right
from collections import Counter, defaultdict
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Any

ROOT = Path(__file__).resolve().parents[1]
DEFAULT_CANDIDATES = ROOT / 'candidates' / 'candidates.json'
DEFAULT_REVIEWS = ROOT / 'reviews' / 'reviews.json'
DEFAULT_QUEUE = ROOT / 'reviews' / 'manual_replay_queue.json'
DEFAULT_REPORT = ROOT / 'reports' / 'deterministic_review_summary.json'


def now_iso() -> str:
    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z')


def parse_dt(value: Any) -> datetime:
    if isinstance(value, (int, float)):
        seconds = value / 1000 if value > 10_000_000_000 else value
        return datetime.fromtimestamp(seconds, tz=timezone.utc).replace(microsecond=0)
    text = str(value).replace('Z', '+00:00')
    dt = datetime.fromisoformat(text)
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(timezone.utc).replace(microsecond=0)


def dt_iso(dt: datetime) -> str:
    return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z')


def load_json(path: Path) -> Any:
    return json.loads(path.read_text(encoding='utf-8'))


def rows_from_payload(payload: Any) -> list[Any]:
    if isinstance(payload, list):
        return payload
    if isinstance(payload, dict):
        for key in ('bars', 'ohlcv', 'data', 'candles', 'values'):
            if isinstance(payload.get(key), list):
                return payload[key]
    raise ValueError('OHLCV payload must contain bars array')


def load_bars(tf: str) -> tuple[list[datetime], list[dict[str, Any]]]:
    path = ROOT / 'data' / f'ohlcv_{tf}.json'
    payload = load_json(path)
    rows = rows_from_payload(payload)
    parsed = []
    for row in rows:
        if not isinstance(row, dict):
            raise ValueError(f'{tf} row must be object')
        t = parse_dt(row.get('timestamp', row.get('time', row.get('date'))))
        parsed.append((t, {
            'timestamp': dt_iso(t),
            'open': float(row['open']),
            'high': float(row['high']),
            'low': float(row['low']),
            'close': float(row['close']),
        }))
    parsed.sort(key=lambda x: x[0])
    return [x[0] for x in parsed], [x[1] for x in parsed]


def bar_at_or_before(times: list[datetime], bars: list[dict[str, Any]], dt: datetime) -> int | None:
    idx = bisect_right(times, dt) - 1
    return idx if idx >= 0 else None


def future_indices(times: list[datetime], start: datetime, end: datetime) -> tuple[int, int]:
    a = bisect_right(times, start)
    b = bisect_right(times, end)
    return a, b


def classify_candidate(candidate: dict[str, Any], ctx: dict[str, tuple[list[datetime], list[dict[str, Any]]]]) -> dict[str, Any]:
    cid = candidate['id']
    dt = parse_dt(candidate['timestamp'])
    direction = candidate.get('direction')
    score = float(candidate.get('candidate_score') or 0)
    notes: list[str] = []
    metrics: dict[str, Any] = {'candidate_time': dt_iso(dt), 'direction': direction, 'scanner_score': score}

    t15, b15 = ctx['M15']
    t5, b5 = ctx['M5']
    i15 = bar_at_or_before(t15, b15, dt)
    i5 = bar_at_or_before(t5, b5, dt)
    if i15 is None or i15 < 1 or i5 is None or i5 < 1:
        return review(cid, 'needs_more_context', 'insufficient_prior_bars', None, notes + ['Not enough prior M15/M5 bars for chronological validation.'], metrics)

    cur15, prev15 = b15[i15], b15[i15 - 1]
    cur5, prev5 = b5[i5], b5[i5 - 1]
    if direction == 'Long':
        m15_sweep = cur15['low'] < prev15['low'] and cur15['close'] > prev15['low']
        m5_sweep = cur5['low'] < prev5['low'] and cur5['close'] > prev5['low']
        local_high = max(x['high'] for x in b5[max(0, i5 - 20):i5 + 1])
        sweep_extreme = min(cur15['low'], cur5['low'])
        sign = 1
    else:
        m15_sweep = cur15['high'] > prev15['high'] and cur15['close'] < prev15['high']
        m5_sweep = cur5['high'] > prev5['high'] and cur5['close'] < prev5['high']
        local_low = min(x['low'] for x in b5[max(0, i5 - 20):i5 + 1])
        sweep_extreme = max(cur15['high'], cur5['high'])
        sign = -1
    metrics.update({'m15_sweep_reclaim': m15_sweep, 'm5_sweep_reclaim': m5_sweep, 'sweep_extreme': round(sweep_extreme, 3)})
    if not (m15_sweep and m5_sweep):
        return review(cid, 'rejected', 'scanner_signal_not_confirmed_on_m15_m5', None, ['Prior-bar sweep/reclaim was not confirmed on both M15 and M5 at candidate time.'], metrics)

    # HTF context requirement: need at least prior D/H4 bar and preferably enough history.
    ht_ok = True
    for tf, min_idx in [('D', 2), ('H4', 6), ('H1', 12)]:
        times, bars = ctx[tf]
        idx = bar_at_or_before(times, bars, dt)
        metrics[f'{tf}_prior_index'] = idx
        if idx is None or idx < min_idx:
            ht_ok = False
    if not ht_ok:
        notes.append('HTF context insufficient for confirmed trade; early dataset or not enough D/H4/H1 history.')

    # Confirmation after candidate: first M5 close beyond local structure within 3h.
    a, b = future_indices(t5, dt, dt + timedelta(hours=3))
    future = list(range(a, b))
    confirm_idx = None
    if direction == 'Long':
        metrics['pre_local_high'] = round(local_high, 3)
        for j in future:
            if b5[j]['close'] > local_high and b5[j]['high'] > local_high:
                confirm_idx = j
                impulse_extreme = b5[j]['high']
                break
    else:
        metrics['pre_local_low'] = round(local_low, 3)
        for j in future:
            if b5[j]['close'] < local_low and b5[j]['low'] < local_low:
                confirm_idx = j
                impulse_extreme = b5[j]['low']
                break
    if confirm_idx is None:
        return review(cid, 'rejected' if ht_ok else 'needs_more_context', 'no_structure_confirmation_within_3h', None, notes + ['No M5 close beyond local structure within 3h after sweep.'], metrics)
    confirm_time = t5[confirm_idx]
    metrics['confirmation_time'] = dt_iso(confirm_time)
    metrics['impulse_extreme'] = round(impulse_extreme, 3)

    # Aryy model: require retrace into 50-62% zone after confirmation before target/invalid.
    if direction == 'Long':
        fib50 = impulse_extreme - (impulse_extreme - sweep_extreme) * 0.50
        fib62 = impulse_extreme - (impulse_extreme - sweep_extreme) * 0.62
        zone_low, zone_high = min(fib50, fib62), max(fib50, fib62)
        # setup target: break high + same risk to first conservative liquidity (>=1R)
        invalid = sweep_extreme
        entry = (zone_low + zone_high) / 2
        risk = entry - invalid
    else:
        fib50 = impulse_extreme + (sweep_extreme - impulse_extreme) * 0.50
        fib62 = impulse_extreme + (sweep_extreme - impulse_extreme) * 0.62
        zone_low, zone_high = min(fib50, fib62), max(fib50, fib62)
        invalid = sweep_extreme
        entry = (zone_low + zone_high) / 2
        risk = invalid - entry
    metrics.update({'fib50': round(fib50, 3), 'fib62': round(fib62, 3), 'entry_mid': round(entry, 3), 'invalid': round(invalid, 3), 'risk_points': round(risk, 3)})
    if risk <= 0:
        return review(cid, 'rejected', 'invalid_risk_geometry', None, notes + ['Invalid risk geometry after Fib calculation.'], metrics)

    a2, b2 = future_indices(t5, confirm_time, confirm_time + timedelta(hours=6))
    touch_idx = None
    invalid_before_touch = False
    for j in range(a2, b2):
        bar = b5[j]
        if direction == 'Long':
            if bar['low'] <= invalid:
                invalid_before_touch = True; break
            if bar['low'] <= zone_high and bar['high'] >= zone_low:
                touch_idx = j; break
        else:
            if bar['high'] >= invalid:
                invalid_before_touch = True; break
            if bar['high'] >= zone_low and bar['low'] <= zone_high:
                touch_idx = j; break
    metrics['invalid_before_fib_touch'] = invalid_before_touch
    if touch_idx is None:
        return review(cid, 'shadow', 'observed_no_fib_50_62_entry', None, notes + ['Structure moved after sweep, but no retrace into Fib 50/62 zone within 6h after confirmation.'], metrics)

    touch_time = t5[touch_idx]
    metrics['fib_touch_time'] = dt_iso(touch_time)
    # Outcome simulation after entry is for triage only; not final backtest proof.
    target1 = entry + sign * risk
    target2 = entry + sign * risk * 2
    metrics.update({'target1': round(target1, 3), 'target2': round(target2, 3)})
    outcome = 'unresolved_mechanical'
    triage_r = 0.0
    a3, b3 = future_indices(t5, touch_time, touch_time + timedelta(hours=12))
    for j in range(a3, b3):
        bar = b5[j]
        if direction == 'Long':
            hit_sl = bar['low'] <= invalid
            hit_t2 = bar['high'] >= target2
            hit_t1 = bar['high'] >= target1
        else:
            hit_sl = bar['high'] >= invalid
            hit_t2 = bar['low'] <= target2
            hit_t1 = bar['low'] <= target1
        if hit_sl and hit_t1:
            outcome = 'ambiguous_same_bar'; triage_r = 0.0; break
        if hit_sl:
            outcome = 'mechanical_sl_first'; triage_r = -1.0; break
        if hit_t2:
            outcome = 'mechanical_tp2_first'; triage_r = 2.0; break
        if hit_t1:
            outcome = 'mechanical_tp1_first'; triage_r = 1.0; break
    metrics['mechanical_outcome'] = outcome

    if not ht_ok:
        return review(cid, 'needs_more_context', 'fib_entry_found_but_htf_context_insufficient', triage_r, notes + ['Fib 50/62 entry condition found, but HTF narrative must be manually checked before acceptance.'], metrics)
    return review(cid, 'needs_manual_replay', 'mechanical_fib_entry_candidate', triage_r, notes + ['Passes mechanical sweep + structure confirmation + Fib 50/62 retrace filter. Requires TradingView Bar Replay before any journal promotion.'], metrics)


def review(candidate_id: str, status: str, outcome: str, triage_r: float | None, notes: list[str], metrics: dict[str, Any]) -> dict[str, Any]:
    return {
        'candidate_id': candidate_id,
        'status': status,
        'outcome': outcome,
        'r': triage_r,
        'journal_trade_id': None,
        'notes': ' '.join(notes),
        'metrics': metrics,
        'requires_manual_replay': status in {'needs_manual_replay', 'needs_more_context'},
        'is_confirmed_trade': False,
    }


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument('--candidates', type=Path, default=DEFAULT_CANDIDATES)
    ap.add_argument('--reviews', type=Path, default=DEFAULT_REVIEWS)
    ap.add_argument('--queue', type=Path, default=DEFAULT_QUEUE)
    ap.add_argument('--report', type=Path, default=DEFAULT_REPORT)
    ap.add_argument('--limit', type=int, default=0, help='Optional max candidates')
    args = ap.parse_args()

    ctx = {tf: load_bars(tf) for tf in ['M5', 'M15', 'M30', 'H1', 'H4', 'D']}
    payload = load_json(args.candidates)
    candidates = payload.get('candidates', [])
    if args.limit:
        candidates = candidates[:args.limit]
    reviews = []
    for c in candidates:
        reviews.append(classify_candidate(c, ctx))

    ts = now_iso()
    counts = Counter(r['status'] for r in reviews)
    outcomes = Counter(r['outcome'] for r in reviews)
    manual = [r for r in reviews if r['status'] in {'needs_manual_replay', 'needs_more_context'}]
    # Keep queue concise: highest quality first, but chronological within status buckets.
    status_rank = {'needs_manual_replay': 0, 'needs_more_context': 1}
    manual_sorted = sorted(manual, key=lambda r: (status_rank.get(r['status'], 9), r['metrics'].get('candidate_time', '')))

    out = {
        'metadata': {
            'symbol': 'XAUUSD',
            'mode': 'deterministic_triage_reviews',
            'source_candidates': str(args.candidates.relative_to(ROOT)) if args.candidates.is_relative_to(ROOT) else str(args.candidates),
            'generated_at': ts,
            'review_count': len(reviews),
            'isFinalBacktest': False,
            'notes': 'Mechanical triage only. No confirmed journal trades are created by this script.',
        },
        'reviews': reviews,
    }
    args.reviews.parent.mkdir(parents=True, exist_ok=True)
    args.reviews.write_text(json.dumps(out, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')

    queue = {
        'metadata': {
            'symbol': 'XAUUSD', 'purpose': 'manual_replay_queue_after_deterministic_triage',
            'generated_at': ts, 'candidate_count': len(manual_sorted),
            'notes': 'Review these chronologically in TradingView Bar Replay before promoting any trade.'
        },
        'items': [
            {
                'order': i + 1,
                'candidate_id': r['candidate_id'],
                'status': r['status'],
                'outcome': r['outcome'],
                'timestamp': r['metrics'].get('candidate_time'),
                'direction': r['metrics'].get('direction'),
                'score': r['metrics'].get('scanner_score'),
                'fib_touch_time': r['metrics'].get('fib_touch_time'),
                'mechanical_r': r.get('r'),
                'reason': r['notes'],
            }
            for i, r in enumerate(manual_sorted)
        ],
    }
    args.queue.parent.mkdir(parents=True, exist_ok=True)
    args.queue.write_text(json.dumps(queue, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')

    rvals = [r['r'] for r in reviews if isinstance(r.get('r'), (int, float))]
    report = {
        'generated_at': ts,
        'total_reviewed': len(reviews),
        'status_counts': dict(counts),
        'outcome_counts': dict(outcomes),
        'manual_queue_count': len(manual_sorted),
        'mechanical_r_count': len(rvals),
        'mechanical_r_avg': round(statistics.mean(rvals), 3) if rvals else None,
        'top_manual_queue_preview': queue['items'][:20],
    }
    args.report.parent.mkdir(parents=True, exist_ok=True)
    args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')

    print('reviewed', len(reviews))
    print('status_counts', dict(counts))
    print('outcome_counts_top', dict(outcomes.most_common(10)))
    print('manual_queue_count', len(manual_sorted))
    print('reviews_file', args.reviews)
    print('queue_file', args.queue)
    print('report_file', args.report)
    return 0


if __name__ == '__main__':
    raise SystemExit(main())