#!/usr/bin/env python3
"""V4 programmable ICT/SMC scanner.

Adds minimal requested refinement:
- 50% Fib: keep v3 strong PD-array rule.
- 62% Fib: require FVG or support/resistance at/near 62% level.
Adds programmable ICT/SMC strategies for research:
- IFVG Continuation
- BPR Rebalance
- PDH/PDL Raid Reversal
- PWH/PWL Raid Reversal
- OTE Continuation with Displacement

Research candidates only; no journal writes.
"""
from __future__ import annotations
import json, statistics, argparse
from collections import Counter
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Any
ROOT=Path(__file__).resolve().parents[1]
OUT=ROOT/'candidates/candidates_v4_programmable_ict.json'
VERSION='0.4.0-programmable-ict-smc'
def parse_dt(v):
    dt=datetime.fromisoformat(str(v).replace('Z','+00:00'))
    if dt.tzinfo is None: dt=dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(timezone.utc)
def iso(dt): return dt.replace(microsecond=0).isoformat().replace('+00:00','Z')
class B:
    __slots__=('ts','open','high','low','close')
    def __init__(self,ts,o,h,l,c): self.ts=ts; self.open=o; self.high=h; self.low=l; self.close=c
    @property
    def dt(self): return parse_dt(self.ts)
    @property
    def body(self): return abs(self.close-self.open)
    @property
    def bull(self): return self.close>self.open
    @property
    def bear(self): return self.close<self.open

def load(tf):
    obj=json.loads((ROOT/'data'/f'ohlcv_{tf}.json').read_text()); rows=obj.get('bars',obj) if isinstance(obj,dict) else obj; out=[]
    for r in rows:
        t=r.get('timestamp',r.get('time',r.get('date'))) if isinstance(r,dict) else r[0]
        vals=r if isinstance(r,dict) else {'open':r[1],'high':r[2],'low':r[3],'close':r[4]}
        out.append(B(iso(parse_dt(t)),float(vals['open']),float(vals['high']),float(vals['low']),float(vals['close'])))
    return sorted(out,key=lambda b:b.dt)
def avg_body(bs): return statistics.mean([b.body for b in bs]) if bs else 0
def session(dt):
    h=dt.hour
    if 0<=h<7: return 'Asia'
    if 7<=h<10: return 'London'
    if 12<=h<17: return 'NY AM'
    if 15<=h<17: return 'London Close'
    if 17<=h<21: return 'NY PM'
    return 'Other'
def window(bs,dt,before,after):
    import bisect
    times=[b.dt for b in bs]; i=bisect.bisect_right(times,dt)-1
    return bs[max(0,i-before+1):i+1], bs[i+1:min(len(bs),i+1+after)], i
def prev_levels(daily,weekly,dt):
    pd=[b for b in daily if b.dt.date()<dt.date()]
    monday=dt.date()-timedelta(days=dt.weekday())
    pw=[b for b in weekly if b.dt.date()<monday]
    return {'PDH':pd[-1].high if pd else None,'PDL':pd[-1].low if pd else None,'PWH':pw[-1].high if pw else None,'PWL':pw[-1].low if pw else None}
def htf(daily,h4,h1,weekly,dt):
    dpre,_,_=window(daily,dt,10,0); h4pre,_,_=window(h4,dt,24,0); h1pre,_,_=window(h1,dt,48,0)
    lv=prev_levels(daily,weekly,dt); cur=h1pre[-1] if h1pre else None
    if not cur: return {'bias':'unknown','mode':'unknown','levels':lv,'external_refs':[],'note':'insufficient HTF'}
    tol=max((cur.high-cur.low)*2,5.0); refs=[k for k,v in lv.items() if v is not None and (abs(cur.close-v)<=tol or cur.high>=v>=cur.low)]
    ds=(dpre[-1].close-dpre[0].close) if len(dpre)>1 else 0; hs=(h4pre[-1].close-h4pre[0].close) if len(h4pre)>1 else 0
    bias='bullish' if ds>0 and hs>=0 else 'bearish' if ds<0 and hs<=0 else 'range'
    mode='external_to_internal' if refs else 'internal_to_external'
    return {'bias':bias,'mode':mode,'levels':lv,'external_refs':refs,'note':f'{mode}; bias={bias}; refs={refs or ["internal"]}'}
def pd_arrays(m15,i):
    arr=[]
    for j in range(max(1,i-14),min(len(m15)-1,i+10)):
        prev,cur,nxt=m15[j-1],m15[j],m15[j+1]; base=avg_body(m15[max(0,j-8):j])
        if nxt.low>prev.high and cur.bull and cur.body>base*1.15: arr.append({'type':'bullish_fvg','low':prev.high,'high':nxt.low,'strength':2})
        if nxt.high<prev.low and cur.bear and cur.body>base*1.15: arr.append({'type':'bearish_fvg','low':nxt.high,'high':prev.low,'strength':2})
        if cur.bull and m15[j-1].bear and cur.close>max(x.high for x in m15[max(0,j-6):j]): arr.append({'type':'bullish_ob','low':m15[j-1].low,'high':m15[j-1].high,'strength':2})
        if cur.bear and m15[j-1].bull and cur.close<min(x.low for x in m15[max(0,j-6):j]): arr.append({'type':'bearish_ob','low':m15[j-1].low,'high':m15[j-1].high,'strength':2})
    return arr
def has_pd(level,arr,direction):
    want='bullish' if direction=='Long' else 'bearish'
    return [a for a in arr if want in a['type'] and a['low']<=level<=a['high']]
def sr_at(level,pre,tol=3.0):
    # Support/resistance proxy: level close to repeated swing highs/lows in prior M15 window.
    touches=0
    for b in pre[-40:]:
        if abs(b.high-level)<=tol or abs(b.low-level)<=tol: touches+=1
    return touches>=2
def mk(ts,strategy,direction,score,flags,levels,notes,htfctx):
    dt=parse_dt(ts)
    return {'id':f"candidate_v4_{ts.replace(':','-')}_{direction.lower()}_{strategy.lower().replace(' + ','_').replace(' ','_').replace('/','_')}",'date':dt.date().isoformat(),'timestamp':ts,'symbol':'XAUUSD','strategy':strategy,'direction':direction,'session':session(dt),'timeframe':'D/H4/H1 narrative + M15/M5 entry','entry_timeframe':'M15/M5','candidate_score':round(score,2),'status':'unreviewed','requires_manual_replay':True,'scanner_flags':flags,'levels':levels,'htf_narrative':htfctx,'notes':notes,'isFinalBacktest':False}
def scan(m15,daily,h4,h1,weekly):
    out=[]
    for i in range(40,len(m15)-16):
        b=m15[i]; pre=m15[max(0,i-24):i]; fut=m15[i+1:i+12]; ctx=htf(daily,h4,h1,weekly,b.dt); arr=pd_arrays(m15,i); base=avg_body(pre)
        disp=b.body>base*1.5 if base else False
        # Main Fib model: M15 impulse, M15/M5 pullback entry.
        if disp and ((b.bull and ctx['bias'] in ('bullish','range')) or (b.bear and ctx['bias'] in ('bearish','range'))):
            direction='Long' if b.bull else 'Short'; hi,lo=b.high,b.low
            f50=hi-(hi-lo)*0.5 if direction=='Long' else lo+(hi-lo)*0.5
            f62=hi-(hi-lo)*0.62 if direction=='Long' else lo+(hi-lo)*0.62
            pd50=has_pd(f50,arr,direction); pd62=has_pd(f62,arr,direction); sr62=sr_at(f62,pre)
            touch50=any(x.low<=f50<=x.high for x in fut); touch62=any(x.low<=f62<=x.high for x in fut)
            if touch50 and pd50:
                out.append(mk(b.ts,'Aryy HTF Narrative Fib 50/62',direction,84,['htf_narrative',ctx['mode'],'fib50_strong_pd_array','market_structure'],{'entry':f50,'stop':lo if direction=='Long' else hi,'fib_entry_level':'50%','planned_rr':2.0,'pd_arrays':pd50},'50% Fib with strong PD Array at level; structure-following entry.',ctx))
            if touch62 and (pd62 or sr62):
                out.append(mk(b.ts,'Aryy HTF Narrative Fib 50/62',direction,80,['htf_narrative',ctx['mode'],'fib62_fvg_or_sr','market_structure'],{'entry':f62,'stop':lo if direction=='Long' else hi,'fib_entry_level':'62%','planned_rr':3.0,'pd_arrays':pd62,'sr_at_level':sr62},'62% Fib allowed only with FVG or support/resistance at level.',ctx))
        # Other programmable strategies.
        pre_high=max(x.high for x in pre); pre_low=min(x.low for x in pre)
        bull_sweep=b.low<min(x.low for x in pre[-6:]) and b.close>pre[-1].low
        bear_sweep=b.high>max(x.high for x in pre[-6:]) and b.close<pre[-1].high
        bullpd=[a for a in arr if 'bullish' in a['type']]; bearpd=[a for a in arr if 'bearish' in a['type']]
        if bull_sweep and b.bull and bullpd and ctx['bias'] in ('bullish','range'):
            a=bullpd[0]; out.append(mk(b.ts,'ICT Sweep + MSS + FVG','Long',74,['sellside_sweep','mss_proxy','pd_array','htf_context'],{'entry':(a['low']+a['high'])/2,'stop':b.low,'planned_rr':2.0,'pd_arrays':bullpd[:2]},'Sweep + MSS/FVG with HTF context.',ctx))
        if bear_sweep and b.bear and bearpd and ctx['bias'] in ('bearish','range'):
            a=bearpd[0]; out.append(mk(b.ts,'ICT Sweep + MSS + FVG','Short',74,['buyside_sweep','mss_proxy','pd_array','htf_context'],{'entry':(a['low']+a['high'])/2,'stop':b.high,'planned_rr':2.0,'pd_arrays':bearpd[:2]},'Sweep + MSS/FVG with HTF context.',ctx))
        if b.low<pre_low and any(x.close>pre_high for x in fut) and bullpd and ctx['bias'] in ('bullish','range'):
            a=bullpd[0]; out.append(mk(b.ts,'Breaker Continuation','Long',72,['sellside_sweep','break_opposite_structure','breaker_pd_retest'],{'entry':(a['low']+a['high'])/2,'stop':b.low,'planned_rr':2.0,'pd_arrays':bullpd[:2]},'Breaker continuation, clean sweep-break-retest.',ctx))
        if b.high>pre_high and any(x.close<pre_low for x in fut) and bearpd and ctx['bias'] in ('bearish','range'):
            a=bearpd[0]; out.append(mk(b.ts,'Breaker Continuation','Short',72,['buyside_sweep','break_opposite_structure','breaker_pd_retest'],{'entry':(a['low']+a['high'])/2,'stop':b.high,'planned_rr':2.0,'pd_arrays':bearpd[:2]},'Breaker continuation, clean sweep-break-retest.',ctx))
        if bull_sweep and ctx['external_refs'] and ctx['bias'] in ('bullish','range'):
            out.append(mk(b.ts,'Liquidity Sweep Reversal','Long',74,['sellside_sweep','external_level_context','reversal_candidate'],{'entry':b.close,'stop':b.low,'planned_rr':2.0},'External level raid reversal.',ctx))
        if bear_sweep and ctx['external_refs'] and ctx['bias'] in ('bearish','range'):
            out.append(mk(b.ts,'Liquidity Sweep Reversal','Short',74,['buyside_sweep','external_level_context','reversal_candidate'],{'entry':b.close,'stop':b.high,'planned_rr':2.0},'External level raid reversal.',ctx))
        # New strategies: IFVG Continuation (FVG fails/inverts then retest)
        if len(arr)>=2:
            for a in arr[:2]:
                mid=(a['low']+a['high'])/2
                if 'bullish' in a['type'] and any(x.close<a['low'] for x in fut[:4]) and ctx['bias'] in ('bearish','range'):
                    out.append(mk(b.ts,'IFVG Continuation','Short',68,['ifvg','failed_bullish_fvg','continuation'],{'entry':mid,'stop':a['high'],'planned_rr':2.0,'pd_arrays':[a]},'Bullish FVG inverted; bearish continuation retest.',ctx))
                if 'bearish' in a['type'] and any(x.close>a['high'] for x in fut[:4]) and ctx['bias'] in ('bullish','range'):
                    out.append(mk(b.ts,'IFVG Continuation','Long',68,['ifvg','failed_bearish_fvg','continuation'],{'entry':mid,'stop':a['low'],'planned_rr':2.0,'pd_arrays':[a]},'Bearish FVG inverted; bullish continuation retest.',ctx))
        # BPR Rebalance: overlapping opposing FVG zones.
        bulls=[a for a in arr if a['type']=='bullish_fvg']; bears=[a for a in arr if a['type']=='bearish_fvg']
        for bu in bulls[:1]:
            for be in bears[:1]:
                lo=max(bu['low'],be['low']); hi=min(bu['high'],be['high'])
                if lo<hi:
                    mid=(lo+hi)/2; direction='Long' if ctx['bias']!='bearish' else 'Short'; stop=lo if direction=='Long' else hi
                    out.append(mk(b.ts,'BPR Rebalance',direction,67,['bpr','opposing_fvg_overlap','rebalance'],{'entry':mid,'stop':stop,'planned_rr':2.0,'bpr_low':lo,'bpr_high':hi},'Balanced Price Range rebalance candidate.',ctx))
        # PDH/PDL and PWH/PWL explicit raid reversals.
        lv=ctx['levels']
        for name in ['PDH','PDL','PWH','PWL']:
            level=lv.get(name)
            if level is None: continue
            if b.high>level and b.close<level and ctx['bias'] in ('bearish','range'):
                out.append(mk(b.ts,f'{name} Raid Reversal','Short',70,[name.lower()+'_raid','external_liquidity','reversal'],{'entry':b.close,'stop':b.high,'planned_rr':2.0,'level':level},f'{name} raid and close back below.',ctx))
            if b.low<level and b.close>level and ctx['bias'] in ('bullish','range'):
                out.append(mk(b.ts,f'{name} Raid Reversal','Long',70,[name.lower()+'_raid','external_liquidity','reversal'],{'entry':b.close,'stop':b.low,'planned_rr':2.0,'level':level},f'{name} raid and close back above.',ctx))
        # OTE continuation: structure impulse, retrace into 62-79 zone with displacement.
        if disp:
            direction='Long' if b.bull else 'Short'; hi,lo=b.high,b.low
            ote62=hi-(hi-lo)*0.62 if direction=='Long' else lo+(hi-lo)*0.62
            ote79=hi-(hi-lo)*0.79 if direction=='Long' else lo+(hi-lo)*0.79
            touch=any(x.low<=max(ote62,ote79) and x.high>=min(ote62,ote79) for x in fut)
            if touch and ((direction=='Long' and ctx['bias'] in ('bullish','range')) or (direction=='Short' and ctx['bias'] in ('bearish','range'))):
                out.append(mk(b.ts,'OTE Continuation with Displacement',direction,69,['ote_62_79','displacement','continuation'],{'entry':(ote62+ote79)/2,'stop':lo if direction=='Long' else hi,'planned_rr':2.5,'ote62':ote62,'ote79':ote79},'OTE continuation after displacement.',ctx))
    # dedupe
    seen=set(); clean=[]
    for c in sorted(out,key=lambda x:(x['timestamp'],x['strategy'],x['direction'],x['levels'].get('fib_entry_level',''))):
        k=(c['timestamp'],c['strategy'],c['direction'],round(float(c['levels'].get('entry',0)),3),c['levels'].get('fib_entry_level',''))
        if k in seen: continue
        seen.add(k); clean.append(c)
    return clean
def main():
    ap=argparse.ArgumentParser(); ap.add_argument('--output',type=Path,default=OUT); args=ap.parse_args()
    c=scan(load('M15'),load('D'),load('H4'),load('H1'),load('W'))
    meta={'symbol':'XAUUSD','mode':'v4_programmable_ict_smc_scan','version':VERSION,'generated_at':iso(datetime.now(timezone.utc)),'candidate_count':len(c),'strategy_counts':dict(Counter(x['strategy'] for x in c)),'notes':'Research candidates only; includes 62% FVG/SR filter and programmable ICT/SMC strategies.'}
    args.output.parent.mkdir(exist_ok=True,parents=True); args.output.write_text(json.dumps({'metadata':meta,'candidates':c},indent=2,ensure_ascii=False)+'\n')
    print(json.dumps(meta,indent=2))
if __name__=='__main__': main()