
    YjH-                    >   d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
m
Z
mZ ddlmZ ddlmZmZ ddlmZ ddlmZ dd	lmZ  ed
                                          Zedz  Zedz  Zedz  Zedz  Zh dZg dZddgZddgZ ddgZ!g dZ"ddgZ#eeefD ]Z$e$%                    dd           dJdZ&dKd Z'dLd"Z(dMdNd)Z)dOd-Z*dPd/Z+dQd1Z,dRd4Z-dSd8Z.dTd;Z/dSd<Z0dSd=Z1dUdCZ2dVdEZ3 G dF dGe          Z4dWdHZ5e6dIk    r e7 e5                      dS )Xak  XAUUSD backtest static dashboard + raw OHLCV upload server.

Serves the project directory like python -m http.server, plus:
  POST /api/upload-raw

The upload endpoint accepts TradingView CSV exports or OHLCV JSON files, saves the
original file under hybrid/data/raw/, normalizes/merges bars into
hybrid/data/ohlcv_<TF>.json, and returns summary statistics only.
    )annotationsN)datetimetimezone)
HTTPStatus)SimpleHTTPRequestHandlerThreadingHTTPServer)Path)Any)urlparsez9/home/aryy/.hermes/profiles/finance/backtests/xauusd_2026hybriddatarawreports>   DWH1H4M5M15M30)timeztime utcdater   	timestampopenz
open pricehighz
high pricelowz	low price)closezclose pricelastvolumevolT)parentsexist_okreturnstrc                     t          j        t          j                                      d                                                              dd          S Nr   microsecond+00:00Z)r   nowr   utcreplace	isoformat     hybrid/scripts/upload_server.pynow_isor2   -   sC    <%%--!-<<FFHHPPQY[^___r0   namec                    t          | pd          j        }t          j        dd|                              d          pd}t          j        t          j                  	                    d          }| d| S )Nuploadz[^A-Za-z0-9._-]+_z._z%Y%m%dT%H%M%SZ)
r	   r3   resubstripr   r+   r   r,   strftime)r3   basestamps      r1   	slug_namer=   1   sr     !!&D6%sD1177==IDL&&//0@AAEdr0   textc                    t          j        ddt          |                                                                                               dd                    S )Nz\s+ r6   )r7   r8   r$   r9   lowerr-   )r>   s    r1   norm_keyrB   8   sB    6&#s4yy006688@@cJJKKKr0   headers	list[str]aliasesrequiredbool
str | Nonec                r    d | D             }|D ]}||v r
||         c S |rt          d| d|            d S )Nc                .    i | ]}t          |          |S r/   )rB   ).0hs     r1   
<dictcomp>zpick.<locals>.<dictcomp>=   s     ///!x{{A///r0   z%Missing required column. Need one of z; got )
ValueError)rC   rE   rF   mappingaliass        r1   pickrQ   <   sr    //w///G " "G5>!!!  [YYYPWYYZZZ4r0   valuer
   r   c                (   t          | t          t          f          rNt          |           }|dk    r|dz  }t          j        |t
          j                                      d          S t          |           	                                }t          j        d|          rt          t          |                    S |                    dd                              d	d
          }	 t          j        |          }n=# t          $ r0 dD ])}	 t          j        ||          } n# t          $ r Y &w xY w Y nw xY w|j         |                    t
          j                  }|                    t
          j                                      d          S )Nl    d(	 i  )tzr   r'   z\d+(\.\d+)?z UTC r*   r)   )z%Y-%m-%d %H:%M:%Sz%Y-%m-%d %H:%Mz%d/%m/%Y %H:%M:%Sz%d/%m/%Y %H:%Mz%m/%d/%Y %H:%M:%Sz%m/%d/%Y %H:%M)tzinfo)
isinstanceintfloatr   fromtimestampr   r,   r-   r$   r9   r7   	fullmatchparse_dtfromisoformatrN   strptimerV   
astimezone)rR   secondsr>   dtfmts        r1   r\   r\   F   s   %#u&& W,,^##tOG%g(,???GGTUGVVVu::D	|ND)) %d$$$<<##++C::D
#D))    I 	 	C&tS11    	 	 
yZZx|Z,,==&&..1.===s6   (C= =D7D#!D7#
D0-D7/D00D76D7ra   c                    |                      t          j                                      d                                                              dd          S r&   )r_   r   r,   r-   r.   )ra   s    r1   isord   `   sE    ==&&..1.==GGIIQQRZ\_```r0   float | Nonec                    | | dk    rd S t          t          |                               dd                    }t          j        |          st          j        |          rd S |S )NrU   ,)rY   r$   r-   mathisnanisinf)rR   numbers     r1   as_floatrl   d   sd    }t3u::%%c2..//Fz& TZ// tMr0   rowdict[str, Any]c                .   |                      d|                      d|                      d|                      d                                        }|t          d|            t          |          }t          |                                          t          |          t          | d                   t          | d                   t          | d                   t          | d	                   t          |                      d
                    dS )Nr   r   r   r   z'OHLCV row missing timestamp/time/date: r   r   r   r   r   )r   r   r   r   r   r   r   )getrN   r\   rX   r   rd   rY   rl   )rm   r   ra   s      r1   normalize_barrq   m   s    SWWVSWWVSWWZEXEX5Y5Y%Z%Z[[IH3HHIII	)		BBLLNN##WWc&k""c&k""SZ  s7|$$3778,,--  r0   pathr	   list[dict[str, Any]]c                P   g }|                      dd          5 }t          j        |          }|j        pg }t	          |t
                    }t	          |t                    }t	          |t                    }t	          |t                    }t	          |t                    }	t	          |t          d          }
|D ]S}|                    t          ||         ||         ||         ||         ||	         |
r||
         nd d                     T	 d d d            n# 1 swxY w Y   |S )NrU   z	utf-8-sig)newlineencodingF)rF   r   r   r   r   r   r   )r   csv
DictReader
fieldnamesrQ   TIME_ALIASESOPEN_ALIASESHIGH_ALIASESLOW_ALIASESCLOSE_ALIASESVOL_ALIASESappendrq   )rr   barshandlereaderrC   time_colopen_colhigh_collow_col	close_colvol_colrm   s               r1   	parse_csvr   }   si   !#D	2	4	4 ''#)r......w,,-00	we<<< 	 	CKKHHH7|Y*1;#g,,t' '      	              $ Ks   C4DD"Dpayload	list[Any]c                    t          | t                    r| S t          | t                    r3dD ]0}|                     |          }t          |t                    r|c S 1t	          d          )N)r   ohlcvr   candlesvalueszCJSON must be an array or object with bars/ohlcv/data/candles/values)rW   listdictrp   rN   )r   keyrowss      r1   extract_json_rowsr      sz    '4   '4   A 	 	C;;s##D$%% 
Z
[
[[r0   c                4   t          j        |                     d                    }g }t          |          D ]}t	          |t
                    r#|                    t          |                     :t	          |t          t          f          rvt          |          dk    rc|                    t          |d         |d         |d         |d         |d         t          |          dk    r|d         nd d	                     t          d
|          |S )Nutf-8rv      r               rw   zUnsupported OHLCV row shape: )jsonloads	read_textr   rW   r   r   rq   r   tuplelenrN   )rr   r   r   rm   s       r1   
parse_jsonr      s`   j99::G!#D )) F Fc4   	FKKc**++++dE]++ 	FCAKKs1vs1vsSTv^abc^dorstou  LO  PS  LT  LT  WX  LX  LX  BE  FG  BH  BH  ^b  'c  'c  d  d  e  e  e  eDSDDEEEKr0   c                    |                                  sg S t          j        |                     d                    }t	          |          }d |D             S )Nr   r   c                    g | ]u}t          |t                    rt          |          nNt          |d          |d         |d         |d         |d         t          |          dk    r|d         ndd          vS )r   r   r   r   r   r   Nrw   )rW   r   rq   r   )rK   rm   s     r1   
<listcomp>z!load_existing.<locals>.<listcomp>   s     Z  Z  Z  NQ*S$"7"7  IM#]TWXYTZdghidjtwxytz  DG  HI  DJ  UX  YZ  U[  qt  ux  qy  qy  |}  q}  q}  gj  kl  gm  gm  CG  LH  LH  >I  >I  Z  Z  Zr0   )existsr   r   r   r   )rr   r   r   s      r1   load_existingr      sl    ;;== 	j99::GW%%D Z  Z  UY  Z  Z  Z  Zr0   	timeframeoutput_pathimported_countrX   
saved_filec                D   t          j        |                    d                    }|                    dg           }d |D             }d |D             }d| |t	          |          |r|d         d         nd |r|d	         d         nd |r"t          t          j        |          d
          nd |r"t          t          j        |          d
          nd |r"t          t          j        |          d
          nd t          |
                    t                              t          |
                    t                              t                      d}t          |                                  dz                      t          j        |d          dz   d           |S )Nr   r   r   c                d    g | ]-}t          |d                    t          |d                   z
  .S )r   r   )rY   rK   bs     r1   r   zsummarize.<locals>.<listcomp>   s3    ???QeAfIqx0???r0   c           	     ~    g | ]:}t          t          |d                    t          |d                   z
            ;S )r   r   )absrY   r   s     r1   r   zsummarize.<locals>.<listcomp>   s=    FFFAc%'
##eAfI&6&6677FFFr0   Tr   r   r   )okr   imported_bars
total_bars
first_time	last_time	avg_rangemedian_rangeavg_bodyraw_saved_asoutput_file
updated_atz_upload_summary.jsonr   indent
)r   r   r   rp   r   round
statisticsmeanmedianr$   relative_toROOTr2   REPORTSrA   
write_textdumps)	r   r   r   r   r   r   rangesbodiessummarys	            r1   	summarizer      s   j...@@AAG;;vr""D??$???FFFFFFF'$ii.2<d1gk**.2<T"Xk**:@JU:?622A666d?EOj/77;;;49?IE*/&111555TJ2248899;2248899ii G )//##9999EEdjQXabFcFcFcfjFju|E}}}Nr0   modec                  
 |                                  } | t          vr't          d|  dt          t                               |j                                        }|dk    rt          |          }n%|dk    rt          |          }nt          d          t          d|  dz  }|dk    rg nt          |          }d |D             
|D ]}|
t          |d	                   <   
fd
t          
          D             }d| dt          |          |r|d         d         nd |r|d         d         nd t          |                    t                              t                      d|d}	|                    t#          j        |	d          dz   d           t#          j        |                    d                     t+          | |t          |          |          S )NzUnsupported timeframe z. Allowed: z.csvz.jsonz)Only .csv and .json uploads are supportedohlcv_r-   c                :    i | ]}t          |d                    |S )r   )rX   r   s     r1   rM   z merge_upload.<locals>.<dictcomp>   s$    333Qs1V9~~q333r0   r   c                     g | ]
}|         S r/   r/   )rK   kby_times     r1   r   z merge_upload.<locals>.<listcomp>   s    0001GAJ000r0   XAUUSDraw_upload_dashboard_merger   r   r   )symbolr   source	bar_countr   r   last_upload_filelast_updated)metadatar   r   r   r   r   r   )upper
ALLOWED_TFrN   sortedsuffixrA   r   r   DATAr   rX   r   r$   r   r   r2   r   r   r   r   r   r   )r   r   r   r   incomingoutputexistingbarr   objr   s             @r1   merge_uploadr      s   !!I
""\)\\zHZHZ\\]]]$$&&FZ((	7		j))DEEE-Y----FY&&rrM&,A,AH33(333G ( ($'CK  !!0000w000D "2T26@$q'+..D26@b+..D #J$:$:4$@$@ A A#II	
 	
  C djQ///$6IIIJv11222YHzBBBr0   c                  <     e Zd ZdZd fdZddZddZddZ xZS )HandlerzXAUUSDBacktestUpload/1.0argsr
   kwargsr#   Nonec                b     t                      j        |dt          t                    i| d S )N	directory)super__init__r$   r   )selfr   r   	__class__s      r1   r   zHandler.__init__   s0    $>#d))>v>>>>>r0   statusrX   r   rn   c                   t          j        |d                              d          }|                     |           |                     dd           |                     dt          t          |                               |                     dd           |                                  | j        	                    |           d S )	Nr   r   r   Content-Typezapplication/json; charset=utf-8zContent-LengthAccess-Control-Allow-Origin*)
r   r   encodesend_responsesend_headerr$   r   end_headerswfilewrite)r   r   r   bodys       r1   	send_jsonzHandler.send_json   s    z'!,,,33G<<6""")JKKK)3s4yy>>:::6<<<
r0   c                    |                      d           |                     dd           |                     dd           |                     dd           |                                  d S )N   r   r   zAccess-Control-Allow-MethodszPOST, OPTIONSzAccess-Control-Allow-Headersr   )r   r   r   )r   s    r1   
do_OPTIONSzHandler.do_OPTIONS   ss    36<<<7III7HHHr0   c                   t          | j                  }|j        dk    r"|                     t          j        d           d S 	 | j                            dd          }d|d}t          j        | j	        | j        |          }t          |                    dd	                                                    }t          |                    d
d                                                    }|dvrt          d          d|v r|d         nd }|t          |dd           st          d          t           t#          |j                  z  }|                    d          5 }	t)          j        |j        |	           d d d            n# 1 swxY w Y   t/          |||          }
|                     d|
           d S # t2          $ r>}|                     ddt          |          t5                      d           Y d }~d S d }~ww xY w)Nz/api/upload-rawz	Not foundr   rU   POST)REQUEST_METHODCONTENT_TYPE)fprC   environr   r   r   merge>   r  r-   zmode must be merge or replacefilefilenamezfile field is requiredwb   i  F)r   errorr   )r   rr   
send_errorr   	NOT_FOUNDrC   rp   cgiFieldStoragerfiler$   getfirstr   rA   rN   getattrRAWr=   r  r   shutilcopyfileobjr  r   r  	Exceptionr2   )r   parsedcontent_typer
  formr   r   itemsavedr   r   excs               r1   do_POSTzHandler.do_POST  s8   $)$$;+++OOJ0+>>>F	[<++NB??L)/NNG#tz4<QXYYYDDMM+t<<==CCEEIt}}VW5566<<>>D/// !@AAA#)T>>4<<tD|74T#B#B| !9:::)DM222ED!! 6V"49f5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6"9eT::GNN3((((( 	[ 	[ 	[NN3us3xxwyy Y YZZZZZZZZZ	[s=   DF= "F	=F= 	FF= F*F= =
H3H  H)r   r
   r   r
   r#   r   )r   rX   r   rn   r#   r   )r#   r   )	__name__
__module____qualname__server_versionr   r  r  r"  __classcell__)r   s   @r1   r   r      s        /N? ? ? ? ? ?      [ [ [ [ [ [ [ [r0   r   c            	        dd l } |                                 }|                    dd           |                    dt          d           |                                }t          |j        |j        ft                    5 }t          dt           d	|j         d
|j         d           |                                 d d d            n# 1 swxY w Y   dS )Nr   z--hostz0.0.0.0)defaultz--porti="  )typer)  zServing z with upload API on http://:T)flush)argparseArgumentParseradd_argumentrX   
parse_argsr   hostportr   printr   serve_forever)r-  parserr   httpds       r1   mainr7    s   OOO$$&&F
)444
sD999D	di3W	=	= QQQ$)QQdiQQY]^^^^               1s   >CCC__main__)r#   r$   )r3   r$   r#   r$   )r>   r$   r#   r$   )T)rC   rD   rE   rD   rF   rG   r#   rH   )rR   r
   r#   r   )ra   r   r#   r$   )rR   r
   r#   re   )rm   rn   r#   rn   )rr   r	   r#   rs   )r   r
   r#   r   )
r   r$   r   r	   r   rX   r   r	   r#   rn   )r   r$   r   r	   r   r$   r#   rn   )r#   rX   )8__doc__
__future__r   r  rx   r   rh   r7   r  r   r   r   httpr   http.serverr   r   pathlibr	   typingr
   urllib.parser   resolver   HYBRIDr   r  r   r   r{   r|   r}   r~   r   r   r   mkdirr2   r=   rB   rQ   r\   rd   rl   rq   r   r   r   r   r   r   r   r7  r#  
SystemExitr/   r0   r1   <module>rD     s?    # " " " " " 



 



   				      ' ' ' ' ' ' ' '       E E E E E E E E             ! ! ! ! ! !tGHHPPRR	
Ul
9
777
DDD%%k"000W% 1 1IOOD4O0000` ` ` `   L L L L    > > > >4a a a a          .\ \ \ \
 
 
 
Z Z Z Z   . C  C  C  CF,[ ,[ ,[ ,[ ,[& ,[ ,[ ,[^	 	 	 	 z
*TTVV

 r0   