#!/usr/bin/env python3
"""XAUUSD backtest static dashboard + raw OHLCV upload server.

Serves the project directory like python -m http.server, plus:
  POST /api/upload-raw

The upload endpoint accepts TradingView CSV exports or OHLCV JSON files, saves the
original file under hybrid/data/raw/, normalizes/merges bars into
hybrid/data/ohlcv_<TF>.json, and returns summary statistics only.
"""
from __future__ import annotations

import cgi
import csv
import json
import math
import re
import shutil
import statistics
from datetime import datetime, timezone
from http import HTTPStatus
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any
from urllib.parse import urlparse

ROOT = Path('/home/aryy/.hermes/profiles/finance/backtests/xauusd_2026').resolve()
HYBRID = ROOT / 'hybrid'
DATA = HYBRID / 'data'
RAW = DATA / 'raw'
REPORTS = HYBRID / 'reports'
ALLOWED_TF = {'M5', 'M15', 'M30', 'H1', 'H4', 'D', 'W'}

TIME_ALIASES = ['time', 'time utc', 'date', 'datetime', 'timestamp']
OPEN_ALIASES = ['open', 'open price']
HIGH_ALIASES = ['high', 'high price']
LOW_ALIASES = ['low', 'low price']
CLOSE_ALIASES = ['close', 'close price', 'last']
VOL_ALIASES = ['volume', 'vol']

for directory in (DATA, RAW, REPORTS):
    directory.mkdir(parents=True, exist_ok=True)


def now_iso() -> str:
    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z')


def slug_name(name: str) -> str:
    base = Path(name or 'upload').name
    base = re.sub(r'[^A-Za-z0-9._-]+', '_', base).strip('._') or 'upload'
    stamp = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
    return f'{stamp}_{base}'


def norm_key(text: str) -> str:
    return re.sub(r'\s+', ' ', str(text).strip().lower().replace('_', ' '))


def pick(headers: list[str], aliases: list[str], required: bool = True) -> str | None:
    mapping = {norm_key(h): h for h in headers}
    for alias in aliases:
        if alias in mapping:
            return mapping[alias]
    if required:
        raise ValueError(f'Missing required column. Need one of {aliases}; got {headers}')
    return None


def parse_dt(value: Any) -> datetime:
    if isinstance(value, (int, float)):
        seconds = float(value)
        if seconds > 10_000_000_000:
            seconds /= 1000
        return datetime.fromtimestamp(seconds, tz=timezone.utc).replace(microsecond=0)
    text = str(value).strip()
    if re.fullmatch(r'\d+(\.\d+)?', text):
        return parse_dt(float(text))
    text = text.replace(' UTC', '').replace('Z', '+00:00')
    try:
        dt = datetime.fromisoformat(text)
    except ValueError:
        for fmt in ['%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%d/%m/%Y %H:%M:%S', '%d/%m/%Y %H:%M', '%m/%d/%Y %H:%M:%S', '%m/%d/%Y %H:%M']:
            try:
                dt = datetime.strptime(text, fmt)
                break
            except ValueError:
                continue
        else:
            raise
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(timezone.utc).replace(microsecond=0)


def iso(dt: datetime) -> str:
    return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z')


def as_float(value: Any) -> float | None:
    if value is None or value == '':
        return None
    number = float(str(value).replace(',', ''))
    if math.isnan(number) or math.isinf(number):
        return None
    return number


def normalize_bar(row: dict[str, Any]) -> dict[str, Any]:
    timestamp = row.get('timestamp', row.get('time', row.get('date', row.get('datetime'))))
    if timestamp is None:
        raise ValueError(f'OHLCV row missing timestamp/time/date: {row}')
    dt = parse_dt(timestamp)
    return {
        'time': int(dt.timestamp()),
        'timestamp': iso(dt),
        'open': float(row['open']),
        'high': float(row['high']),
        'low': float(row['low']),
        'close': float(row['close']),
        'volume': as_float(row.get('volume')),
    }


def parse_csv(path: Path) -> list[dict[str, Any]]:
    bars: list[dict[str, Any]] = []
    with path.open(newline='', encoding='utf-8-sig') as handle:
        reader = csv.DictReader(handle)
        headers = reader.fieldnames or []
        time_col = pick(headers, TIME_ALIASES)
        open_col = pick(headers, OPEN_ALIASES)
        high_col = pick(headers, HIGH_ALIASES)
        low_col = pick(headers, LOW_ALIASES)
        close_col = pick(headers, CLOSE_ALIASES)
        vol_col = pick(headers, VOL_ALIASES, required=False)
        for row in reader:
            bars.append(normalize_bar({
                'time': row[time_col],
                'open': row[open_col],
                'high': row[high_col],
                'low': row[low_col],
                'close': row[close_col],
                'volume': row[vol_col] if vol_col else None,
            }))
    return bars


def extract_json_rows(payload: Any) -> list[Any]:
    if isinstance(payload, list):
        return payload
    if isinstance(payload, dict):
        for key in ('bars', 'ohlcv', 'data', 'candles', 'values'):
            rows = payload.get(key)
            if isinstance(rows, list):
                return rows
    raise ValueError('JSON must be an array or object with bars/ohlcv/data/candles/values')


def parse_json(path: Path) -> list[dict[str, Any]]:
    payload = json.loads(path.read_text(encoding='utf-8'))
    bars: list[dict[str, Any]] = []
    for row in extract_json_rows(payload):
        if isinstance(row, dict):
            bars.append(normalize_bar(row))
        elif isinstance(row, (list, tuple)) and len(row) >= 5:
            bars.append(normalize_bar({'time': row[0], 'open': row[1], 'high': row[2], 'low': row[3], 'close': row[4], 'volume': row[5] if len(row) > 5 else None}))
        else:
            raise ValueError(f'Unsupported OHLCV row shape: {row!r}')
    return bars


def load_existing(path: Path) -> list[dict[str, Any]]:
    if not path.exists():
        return []
    payload = json.loads(path.read_text(encoding='utf-8'))
    rows = extract_json_rows(payload)
    return [normalize_bar(row) if isinstance(row, dict) else normalize_bar({'time': row[0], 'open': row[1], 'high': row[2], 'low': row[3], 'close': row[4], 'volume': row[5] if len(row) > 5 else None}) for row in rows]


def summarize(timeframe: str, output_path: Path, imported_count: int, saved_file: Path) -> dict[str, Any]:
    payload = json.loads(output_path.read_text(encoding='utf-8'))
    bars = payload.get('bars', [])
    ranges = [float(b['high']) - float(b['low']) for b in bars]
    bodies = [abs(float(b['close']) - float(b['open'])) for b in bars]
    summary = {
        'ok': True,
        'timeframe': timeframe,
        'imported_bars': imported_count,
        'total_bars': len(bars),
        'first_time': bars[0]['timestamp'] if bars else None,
        'last_time': bars[-1]['timestamp'] if bars else None,
        'avg_range': round(statistics.mean(ranges), 3) if ranges else None,
        'median_range': round(statistics.median(ranges), 3) if ranges else None,
        'avg_body': round(statistics.mean(bodies), 3) if bodies else None,
        'raw_saved_as': str(saved_file.relative_to(ROOT)),
        'output_file': str(output_path.relative_to(ROOT)),
        'updated_at': now_iso(),
    }
    (REPORTS / f'{timeframe.lower()}_upload_summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8')
    return summary


def merge_upload(timeframe: str, saved_file: Path, mode: str) -> dict[str, Any]:
    timeframe = timeframe.upper()
    if timeframe not in ALLOWED_TF:
        raise ValueError(f'Unsupported timeframe {timeframe}. Allowed: {sorted(ALLOWED_TF)}')
    suffix = saved_file.suffix.lower()
    if suffix == '.csv':
        incoming = parse_csv(saved_file)
    elif suffix == '.json':
        incoming = parse_json(saved_file)
    else:
        raise ValueError('Only .csv and .json uploads are supported')
    output = DATA / f'ohlcv_{timeframe}.json'
    existing = [] if mode == 'replace' else load_existing(output)
    by_time = {int(b['time']): b for b in existing}
    for bar in incoming:
        by_time[int(bar['time'])] = bar
    bars = [by_time[k] for k in sorted(by_time)]
    obj = {
        'metadata': {
            'symbol': 'XAUUSD',
            'timeframe': timeframe,
            'source': 'raw_upload_dashboard_merge',
            'bar_count': len(bars),
            'first_time': bars[0]['timestamp'] if bars else None,
            'last_time': bars[-1]['timestamp'] if bars else None,
            'last_upload_file': str(saved_file.relative_to(ROOT)),
            'last_updated': now_iso(),
        },
        'bars': bars,
    }
    output.write_text(json.dumps(obj, indent=2) + '\n', encoding='utf-8')
    json.loads(output.read_text(encoding='utf-8'))
    return summarize(timeframe, output, len(incoming), saved_file)


class Handler(SimpleHTTPRequestHandler):
    server_version = 'XAUUSDBacktestUpload/1.0'

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        super().__init__(*args, directory=str(ROOT), **kwargs)

    def send_json(self, status: int, payload: dict[str, Any]) -> None:
        body = json.dumps(payload, indent=2).encode('utf-8')
        self.send_response(status)
        self.send_header('Content-Type', 'application/json; charset=utf-8')
        self.send_header('Content-Length', str(len(body)))
        self.send_header('Access-Control-Allow-Origin', '*')
        self.end_headers()
        self.wfile.write(body)

    def do_OPTIONS(self) -> None:
        self.send_response(204)
        self.send_header('Access-Control-Allow-Origin', '*')
        self.send_header('Access-Control-Allow-Methods', 'POST, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type')
        self.end_headers()

    def do_POST(self) -> None:
        parsed = urlparse(self.path)
        if parsed.path != '/api/upload-raw':
            self.send_error(HTTPStatus.NOT_FOUND, 'Not found')
            return
        try:
            content_type = self.headers.get('Content-Type', '')
            environ = {'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': content_type}
            form = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ=environ)
            timeframe = str(form.getfirst('timeframe', 'M5')).upper()
            mode = str(form.getfirst('mode', 'merge')).lower()
            if mode not in {'merge', 'replace'}:
                raise ValueError('mode must be merge or replace')
            item = form['file'] if 'file' in form else None
            if item is None or not getattr(item, 'filename', None):
                raise ValueError('file field is required')
            saved = RAW / slug_name(item.filename)
            with saved.open('wb') as handle:
                shutil.copyfileobj(item.file, handle)
            summary = merge_upload(timeframe, saved, mode)
            self.send_json(200, summary)
        except Exception as exc:
            self.send_json(400, {'ok': False, 'error': str(exc), 'updated_at': now_iso()})


def main() -> int:
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--host', default='0.0.0.0')
    parser.add_argument('--port', type=int, default=8765)
    args = parser.parse_args()
    with ThreadingHTTPServer((args.host, args.port), Handler) as httpd:
        print(f'Serving {ROOT} with upload API on http://{args.host}:{args.port}', flush=True)
        httpd.serve_forever()
    return 0


if __name__ == '__main__':
    raise SystemExit(main())