#!/usr/bin/env python3
"""V3 HTF/PD-array aware candidate scanner for XAUUSD hybrid research.

Fixes issues in v2:
- Main strategy HTF narrative is D/H4/H1 narrative, not M30-only.
- Main entry timeframe is M15/M5.
- Adds PWH/PWL/PDH/PDL context and internal->external / external->internal narrative proxy.
- Fib 50 entry requires a strong PD Array overlap; Fib 62 is allowed with valid structure.
- Other strategies add stricter HTF/context/PD-array checks.

Research output only; does not overwrite journal/backtest-data.
"""
from __future__ import annotations
import json, argparse, statistics
from collections import Counter, defaultdict
from dataclasses import dataclass
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Any

ROOT=Path(__file__).resolve().parents[1]
OUT=ROOT/'candidates/candidates_v3_htf_pdarray.json'
SYMBOL='XAUUSD'; VERSION='0.3.0-htf-pdarray'
STRATEGIES=['Aryy HTF Narrative Fib 50/62','ICT Sweep + MSS + FVG','Breaker Continuation','Liquidity Sweep Reversal']
@dataclass(frozen=True)
class Bar:
    ts:str; open:float; high:float; low:float; close:float
    @property
    def dt(self): return parse_dt(self.ts)
    @property
    def body(self): return abs(self.close-self.open)
    @property
    def bull(self): return self.close>self.open
    @property
    def bear(self): return self.close<self.open

def parse_dt(v:Any)->datetime:
    if isinstance(v,(int,float)): return datetime.fromtimestamp(v if v<1e10 else v/1000,tz=timezone.utc)
    s=str(v).replace('Z','+00:00'); dt=datetime.fromisoformat(s)
    if dt.tzinfo is None: dt=dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(timezone.utc)
def iso(dt): return dt.replace(microsecond=0).isoformat().replace('+00:00','Z')
def load(tf):
    obj=json.loads((ROOT/'data'/f'ohlcv_{tf}.json').read_text())
    rows=obj.get('bars',obj) if isinstance(obj,dict) else obj
    out=[]
    for r in rows:
        t=r.get('timestamp',r.get('time',r.get('date'))) if isinstance(r,dict) else r[0]
        vals=r if isinstance(r,dict) else {'open':r[1],'high':r[2],'low':r[3],'close':r[4]}
        out.append(Bar(iso(parse_dt(t)),float(vals['open']),float(vals['high']),float(vals['low']),float(vals['close'])))
    return sorted(out,key=lambda b:b.dt)
def avg_body(bs): return statistics.mean([b.body for b in bs]) if bs else 0
def session(dt):
    h=dt.hour
    if 0<=h<7: return 'Asia'
    if 7<=h<10: return 'London'
    if 12<=h<17: return 'NY AM'
    if 15<=h<17: return 'London Close'
    if 17<=h<21: return 'NY PM'
    return 'Other'
def window(bs,dt,before,after):
    import bisect
    times=[b.dt for b in bs]; i=bisect.bisect_right(times,dt)-1
    return bs[max(0,i-before+1):i+1], bs[i+1:min(len(bs),i+1+after)]
def prev_day_week_levels(daily, weekly, dt):
    prev_d=[b for b in daily if b.dt.date()<dt.date()]
    pd=prev_d[-1] if prev_d else None
    # previous completed weekly bar before current week
    monday=(dt.date()-timedelta(days=dt.weekday()))
    prev_w=[b for b in weekly if b.dt.date()<monday]
    pw=prev_w[-1] if prev_w else None
    return {'PDH':pd.high if pd else None,'PDL':pd.low if pd else None,'PWH':pw.high if pw else None,'PWL':pw.low if pw else None}
def htf_narrative(daily,h4,h1,dt):
    dpre,_=window(daily,dt,10,0); h4pre,_=window(h4,dt,24,0); h1pre,_=window(h1,dt,48,0)
    levels=prev_day_week_levels(daily,load('W'),dt)
    cur=h1pre[-1] if h1pre else None
    if not cur: return {'bias':'unknown','mode':'unknown','levels':levels,'note':'insufficient HTF'}
    # external if near/through previous day/week levels; otherwise internal.
    tol=max((cur.high-cur.low)*2, 5.0)
    near_ext=[]
    for k,v in levels.items():
        if v is not None and (abs(cur.close-v)<=tol or cur.high>=v>=cur.low): near_ext.append(k)
    d_slope=(dpre[-1].close-dpre[0].close) if len(dpre)>=2 else 0
    h4_slope=(h4pre[-1].close-h4pre[0].close) if len(h4pre)>=2 else 0
    bias='bullish' if d_slope>0 and h4_slope>=0 else 'bearish' if d_slope<0 and h4_slope<=0 else 'range'
    mode='external_to_internal' if near_ext else 'internal_to_external'
    return {'bias':bias,'mode':mode,'levels':levels,'external_refs':near_ext,'note':f'{mode}; bias={bias}; refs={near_ext or ["internal range"]}'}
def pd_arrays(m15,idx):
    arrays=[]
    for i in range(max(1,idx-12), min(len(m15)-1,idx+8)):
        prev=m15[i-1]; cur=m15[i]; nxt=m15[i+1]
        if nxt.low>prev.high and cur.bull and cur.body>avg_body(m15[max(0,i-8):i])*1.2:
            arrays.append({'type':'bullish_fvg','low':prev.high,'high':nxt.low,'strength':2})
        if nxt.high<prev.low and cur.bear and cur.body>avg_body(m15[max(0,i-8):i])*1.2:
            arrays.append({'type':'bearish_fvg','low':nxt.high,'high':prev.low,'strength':2})
        # OB proxy: last opposite candle before displacement
        if i>1 and cur.bull and m15[i-1].bear and cur.close>max(x.high for x in m15[max(0,i-6):i]):
            arrays.append({'type':'bullish_ob','low':m15[i-1].low,'high':m15[i-1].high,'strength':2})
        if i>1 and cur.bear and m15[i-1].bull and cur.close<min(x.low for x in m15[max(0,i-6):i]):
            arrays.append({'type':'bearish_ob','low':m15[i-1].low,'high':m15[i-1].high,'strength':2})
    return arrays
def overlaps(level, arrays, direction):
    valid=[]
    want=('bullish' if direction=='Long' else 'bearish')
    for a in arrays:
        if want in a['type'] and a['low']<=level<=a['high']:
            valid.append(a)
    return valid
def mk(ts,strategy,direction,score,flags,levels,notes,htf):
    dt=parse_dt(ts)
    return {'id':f"candidate_v3_{ts.replace(':','-')}_{direction.lower()}_{strategy.lower().replace(' + ','_').replace(' ','_').replace('/','_')}",'date':dt.date().isoformat(),'timestamp':ts,'symbol':SYMBOL,'strategy':strategy,'direction':direction,'session':session(dt),'timeframe':'HTF narrative + M15/M5 entry','entry_timeframe':'M15/M5','candidate_score':round(score,2),'status':'unreviewed','requires_manual_replay':True,'scanner_flags':flags,'levels':levels,'htf_narrative':htf,'notes':notes,'isFinalBacktest':False}

def scan_main(m15,daily,h4,h1):
    out=[]
    for i in range(30,len(m15)-16):
        b=m15[i]; look=m15[i-20:i]
        if b.body <= avg_body(look)*1.6: continue
        htf=htf_narrative(daily,h4,h1,b.dt); bias=htf['bias']
        if b.bull and bias not in ('bullish','range'): continue
        if b.bear and bias not in ('bearish','range'): continue
        direction='Long' if b.bull else 'Short'
        hi,lo=b.high,b.low
        if direction=='Long': f50=hi-(hi-lo)*0.5; f62=hi-(hi-lo)*0.62
        else: f50=lo+(hi-lo)*0.5; f62=lo+(hi-lo)*0.62
        arrays=pd_arrays(m15,i)
        pd50=overlaps(f50,arrays,direction); pd62=overlaps(f62,arrays,direction)
        # Need future M5/M15 pullback represented by M15 bars touching the levels.
        future=m15[i+1:i+12]
        touch50=any(x.low<=f50<=x.high for x in future); touch62=any(x.low<=f62<=x.high for x in future)
        if touch62:
            out.append(mk(b.ts,'Aryy HTF Narrative Fib 50/62',direction,76,['htf_narrative',htf['mode'],'m15_displacement','fib62_entry'],{'impulse_low':lo,'impulse_high':hi,'fib_entry_level':'62%','entry':f62,'planned_rr':3.0,'pd_arrays':pd62},'62% Fib entry with SL 100%, TP -53%; M15/M5 replay required.',htf))
        if touch50 and pd50:
            out.append(mk(b.ts,'Aryy HTF Narrative Fib 50/62',direction,82,['htf_narrative',htf['mode'],'m15_displacement','fib50_strong_pd_array'],{'impulse_low':lo,'impulse_high':hi,'fib_entry_level':'50%','entry':f50,'planned_rr':2.0,'pd_arrays':pd50},'50% Fib entry allowed because strong PD Array overlaps 50% zone.',htf))
    return out
def scan_other(m15,daily,h4,h1):
    out=[]
    for i in range(20,len(m15)-12):
        b=m15[i]; look=m15[i-12:i]; htf=htf_narrative(daily,h4,h1,b.dt); arrays=pd_arrays(m15,i)
        disp=b.body>avg_body(look)*1.5
        if not disp: continue
        # ICT sweep + MSS + FVG with meaningful PDH/PDL/PWH/PWL or HTF bias alignment.
        bull_sweep=b.low<min(x.low for x in look[-6:]) and b.close>look[-1].low
        bear_sweep=b.high>max(x.high for x in look[-6:]) and b.close<look[-1].high
        bull_pd=[a for a in arrays if 'bullish' in a['type']]; bear_pd=[a for a in arrays if 'bearish' in a['type']]
        if bull_sweep and b.bull and bull_pd and htf['bias'] in ('bullish','range'):
            out.append(mk(b.ts,'ICT Sweep + MSS + FVG','Long',72,['sellside_sweep','mss_proxy','pd_array','htf_'+htf['mode']],{'sweep_low':b.low,'entry':(bull_pd[0]['low']+bull_pd[0]['high'])/2,'stop':b.low,'planned_rr':2.0,'pd_arrays':bull_pd[:2]},'Sweep + MSS/FVG candidate with HTF narrative proxy.',htf))
        if bear_sweep and b.bear and bear_pd and htf['bias'] in ('bearish','range'):
            out.append(mk(b.ts,'ICT Sweep + MSS + FVG','Short',72,['buyside_sweep','mss_proxy','pd_array','htf_'+htf['mode']],{'sweep_high':b.high,'entry':(bear_pd[0]['low']+bear_pd[0]['high'])/2,'stop':b.high,'planned_rr':2.0,'pd_arrays':bear_pd[:2]},'Sweep + MSS/FVG candidate with HTF narrative proxy.',htf))
        # Breaker continuation stricter: requires sweep, opposite break, and PD array retest.
        pre_high=max(x.high for x in look); pre_low=min(x.low for x in look); fut=m15[i+1:i+8]
        if b.low<pre_low and any(x.close>pre_high for x in fut) and bull_pd and htf['bias'] in ('bullish','range'):
            out.append(mk(b.ts,'Breaker Continuation','Long',70,['sellside_sweep','break_opposite_structure','breaker_pd_retest'],{'sweep_low':b.low,'entry':(bull_pd[0]['low']+bull_pd[0]['high'])/2,'stop':b.low,'planned_rr':2.0,'pd_arrays':bull_pd[:2]},'Breaker continuation with failed sellside raid and bullish PD retest.',htf))
        if b.high>pre_high and any(x.close<pre_low for x in fut) and bear_pd and htf['bias'] in ('bearish','range'):
            out.append(mk(b.ts,'Breaker Continuation','Short',70,['buyside_sweep','break_opposite_structure','breaker_pd_retest'],{'sweep_high':b.high,'entry':(bear_pd[0]['low']+bear_pd[0]['high'])/2,'stop':b.high,'planned_rr':2.0,'pd_arrays':bear_pd[:2]},'Breaker continuation with failed buyside raid and bearish PD retest.',htf))
        # Liquidity Sweep Reversal refined with PWH/PWL/PDH/PDL proximity.
        near_ext=bool(htf.get('external_refs'))
        if bull_sweep and near_ext and htf['bias'] in ('bullish','range'):
            out.append(mk(b.ts,'Liquidity Sweep Reversal','Long',74,['sellside_sweep','external_level_context','reversal_candidate'],{'sweep_low':b.low,'entry':b.close,'stop':b.low,'planned_rr':2.0},'External liquidity sweep reversal candidate near PDL/PWL/PDH/PWH.',htf))
        if bear_sweep and near_ext and htf['bias'] in ('bearish','range'):
            out.append(mk(b.ts,'Liquidity Sweep Reversal','Short',74,['buyside_sweep','external_level_context','reversal_candidate'],{'sweep_high':b.high,'entry':b.close,'stop':b.high,'planned_rr':2.0},'External liquidity sweep reversal candidate near PDL/PWL/PDH/PWH.',htf))
    return out
def dedupe(cands):
    seen=set(); out=[]
    for c in sorted(cands,key=lambda x:(x['timestamp'],x['strategy'],x['direction'],x['levels'].get('fib_entry_level',''))):
        k=(c['timestamp'],c['strategy'],c['direction'],c['levels'].get('fib_entry_level',''))
        if k in seen: continue
        seen.add(k); out.append(c)
    return out
def main():
    ap=argparse.ArgumentParser(); ap.add_argument('--output',type=Path,default=OUT); args=ap.parse_args()
    m15=load('M15'); daily=load('D'); h4=load('H4'); h1=load('H1')
    c=dedupe(scan_main(m15,daily,h4,h1)+scan_other(m15,daily,h4,h1))
    meta={'symbol':SYMBOL,'mode':'v3_htf_pdarray_candidate_scan','version':VERSION,'generated_at':iso(datetime.now(timezone.utc)),'candidate_count':len(c),'strategy_counts':dict(Counter(x['strategy'] for x in c)),'notes':'Research candidates only. Corrected HTF narrative + M15/M5 entry + PD-array rules.'}
    args.output.parent.mkdir(parents=True,exist_ok=True); args.output.write_text(json.dumps({'metadata':meta,'candidates':c},indent=2,ensure_ascii=False)+'\n')
    print(json.dumps(meta,indent=2))
if __name__=='__main__': main()