#!/usr/bin/env python3
"""Interactive viewer for public REAL BSCAN files from M2HATS.

This is a small, student-facing adaptation of the REAL reader, preprocessing
sequence, coordinate conversion, and PPI/RHI plotting used in Pierre Derian's
lidar toolbox.  It intentionally does not require the toolbox, CUDA, SSH, or
access to Typhon2/Aeolus2 accounts.

By default it reads one public BSCAN over HTTPS into memory, selects the file
nearest the requested UTC time, displays the scan interactively, and optionally
writes a high-resolution PNG.
"""

from __future__ import annotations

import argparse
import datetime as dt
import io
import re
import sys
import warnings
from html.parser import HTMLParser
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import unquote, urljoin, urlparse
from urllib.request import Request, urlopen
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError

import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
from matplotlib import patches
from scipy.ndimage import median_filter


DEFAULT_BASE_URL = "https://lidar.csuchico.edu/M2HATS-data/REAL/bscans/"
DEFAULT_DATE = dt.date(2023, 8, 4)
DEFAULT_TIME = dt.time(18, 18, 3)
DEFAULT_TITLE = "REAL, Tonopah, NV (M²HATS) - "
DEFAULT_LOW_PASS = 7
DEFAULT_HIGH_PASS = 333
DEFAULT_DB_OFFSET = 500.0
DEFAULT_DB_EPSILON = 1.0e-6
DEFAULT_TICK_STEP = 200.0
DEFAULT_CMAP = "nipy_spectral"
DEFAULT_FILTERED_VMIN = -1.0
DEFAULT_FILTERED_VMAX = 3.0
DEFAULT_UNFILTERED_VMIN = 26.0
DEFAULT_UNFILTERED_VMAX = 40.0
FILENAME_RE = re.compile(r"^REAL\.(\d{8})_(\d{6})\.bscan$")


def parse_date(value: str) -> dt.date:
    try:
        return dt.datetime.strptime(value, "%Y-%m-%d").date()
    except ValueError as exc:
        raise argparse.ArgumentTypeError("date must have the form YYYY-MM-DD") from exc


def parse_time(value: str) -> dt.time:
    for fmt in ("%H:%M:%S", "%H%M%S"):
        try:
            return dt.datetime.strptime(value, fmt).time()
        except ValueError:
            pass
    raise argparse.ArgumentTypeError("time must have the form HH:MM:SS")


def parse_odd_length(value: str) -> int:
    try:
        number = int(value)
    except ValueError as exc:
        raise argparse.ArgumentTypeError("filter length must be an odd integer") from exc
    if number < 1 or number % 2 == 0:
        raise argparse.ArgumentTypeError("filter length must be a positive odd integer")
    return number


def parse_positive_odd_highpass(value: str) -> int:
    number = parse_odd_length(value)
    if number < 3:
        raise argparse.ArgumentTypeError("high-pass length must be at least 3")
    return number


def parse_filename(name: str) -> tuple[dt.datetime, str] | None:
    """Return the naive UTC timestamp and basename for a REAL BSCAN name."""
    name = Path(unquote(urlparse(name).path)).name
    match = FILENAME_RE.fullmatch(name)
    if match is None:
        return None
    return dt.datetime.strptime(
        f"{match.group(1)}_{match.group(2)}", "%Y%m%d_%H%M%S"
    ), name


class DirectoryIndexParser(HTMLParser):
    """Extract links from Apache's directory index without extra packages."""

    def __init__(self) -> None:
        super().__init__()
        self.hrefs: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag.lower() != "a":
            return
        for key, value in attrs:
            if key.lower() == "href" and value:
                self.hrefs.append(value)


def day_directory_url(base_url: str, day: dt.date) -> str:
    base = base_url.rstrip("/") + "/"
    return urljoin(base, f"{day:%Y%m}/{day:%Y%m%d}/")


def local_day_directory(root: Path, day: dt.date) -> Path:
    return root / f"{day:%Y%m}" / f"{day:%Y%m%d}"


def candidate_days(target: dt.datetime) -> list[dt.date]:
    """Include adjacent days so nearest-file selection works around midnight."""
    return [(target + dt.timedelta(days=offset)).date() for offset in (-1, 0, 1)]


def list_local_candidates(root: Path, days: list[dt.date]) -> list[tuple[dt.datetime, str, Path]]:
    candidates: list[tuple[dt.datetime, str, Path]] = []
    for day in days:
        directory = local_day_directory(root, day)
        if not directory.is_dir():
            continue
        for path in directory.glob("REAL.*.bscan"):
            parsed = parse_filename(path.name)
            if parsed is not None:
                timestamp, name = parsed
                candidates.append((timestamp, name, path))
    return candidates


def fetch_url(url: str, timeout: float) -> bytes:
    request = Request(url, headers={"User-Agent": "REAL-M2HATS-viewer/1.0"})
    with urlopen(request, timeout=timeout) as response:
        return response.read()


def list_remote_candidates(
    base_url: str, days: list[dt.date], timeout: float
) -> list[tuple[dt.datetime, str, str]]:
    candidates: list[tuple[dt.datetime, str, str]] = []
    errors: list[str] = []
    for day in days:
        directory_url = day_directory_url(base_url, day)
        try:
            html = fetch_url(directory_url, timeout).decode("utf-8", errors="replace")
        except (HTTPError, URLError, TimeoutError, OSError) as exc:
            errors.append(f"{directory_url} ({exc})")
            continue
        parser = DirectoryIndexParser()
        parser.feed(html)
        for href in parser.hrefs:
            parsed = parse_filename(href)
            if parsed is None:
                continue
            timestamp, name = parsed
            candidates.append((timestamp, name, urljoin(directory_url, href)))
    if not candidates and errors:
        details = "\n".join(f"  - {error}" for error in errors)
        raise RuntimeError(
            "Could not read the public Apache directory listing. "
            "Check the internet connection and the base URL. Tried:\n" + details
        )
    return candidates


def select_nearest(
    candidates: list[tuple[dt.datetime, str, object]], target: dt.datetime
) -> tuple[dt.datetime, str, object]:
    if not candidates:
        raise RuntimeError("No REAL BSCAN files were found near the requested time.")
    return min(candidates, key=lambda item: abs(item[0] - target))


class RealBscanReader:
    """Reader for the two-channel, big-endian REAL BSCAN format."""

    header_size = 30
    id_index = 29
    num_channels = 2
    item_size = 4

    @staticmethod
    def _record_layout(data: bytes) -> tuple[int, int, int]:
        if len(data) < RealBscanReader.header_size * RealBscanReader.item_size:
            raise ValueError("The BSCAN file is too small to contain a REAL header.")
        header = np.frombuffer(
            data, dtype=np.float32, count=RealBscanReader.header_size
        ).byteswap()
        channel_size = int(header[28])
        if channel_size <= 0:
            raise ValueError("The BSCAN header contains an invalid channel length.")
        record_size = RealBscanReader.header_size + RealBscanReader.num_channels * channel_size
        record_bytes = RealBscanReader.item_size * record_size
        num_records = len(data) // record_bytes
        return num_records, record_size, channel_size

    def read_scan(self, data: bytes, scan_index: int = 0) -> dict[str, object]:
        num_records, record_size, channel_size = self._record_layout(data)
        if num_records < 1:
            raise ValueError("The BSCAN file contains no complete records.")

        raw = np.frombuffer(
            data, dtype=np.float32, count=num_records * record_size
        ).byteswap()
        raw = raw.reshape(num_records, record_size)
        ids = raw[:, self.id_index]
        starts = np.r_[0, np.flatnonzero(ids[1:] != ids[:-1]) + 1]
        stops = np.r_[starts[1:], num_records]
        if scan_index < 0 or scan_index >= len(starts):
            raise ValueError(
                f"scan index {scan_index} is unavailable; file contains {len(starts)} scan(s)"
            )
        start, stop = int(starts[scan_index]), int(stops[scan_index])
        records = raw[start:stop]
        headers = records[:, : self.header_size]
        profiles = records[:, self.header_size :].reshape(
            stop - start, self.num_channels, channel_size
        )

        gate_size = float(headers[0, 21])
        if gate_size <= 0:
            raise ValueError("The BSCAN header contains an invalid range-gate size.")
        zero_index = int(-headers[0, 20] / gate_size)
        distances = gate_size * np.arange(-zero_index, channel_size - zero_index)
        azimuths = headers[:, 6].astype(float)
        elevations = headers[:, 7].astype(float)
        timestamps = [self._timestamp(header) for header in headers]

        return {
            "profiles": profiles,
            "distances": distances,
            "azimuths": azimuths,
            "elevations": elevations,
            "timestamps": timestamps,
        }

    @staticmethod
    def _timestamp(header: np.ndarray) -> dt.datetime:
        seconds = int(header[2])
        microseconds = int(1.0e6 * (float(header[2]) - seconds))
        return dt.datetime(
            int(header[13]),
            int(header[11]),
            int(header[12]),
            int(header[0]),
            int(header[1]),
            seconds,
            microseconds,
        )


def scan_label(azimuths: np.ndarray, elevations: np.ndarray) -> str:
    # Unwrap in radians for compatibility with older NumPy releases.
    azimuth_range = np.ptp(np.rad2deg(np.unwrap(np.deg2rad(azimuths))))
    elevation_range = np.ptp(np.rad2deg(np.unwrap(np.deg2rad(elevations))))
    if azimuth_range < 0.015 and elevation_range < 0.015:
        return "STARE"
    return "PPI" if azimuth_range > elevation_range else "RHI"


def polar_grid(distances: np.ndarray, angles: np.ndarray, meteo: bool) -> tuple[np.ndarray, np.ndarray]:
    ranges = distances.reshape(1, -1)
    radians = np.deg2rad(angles).reshape(-1, 1)
    if meteo:
        return np.sin(radians) * ranges, np.cos(radians) * ranges
    return np.cos(radians) * ranges, np.sin(radians) * ranges


def preprocess(
    raw: dict[str, object],
    processing: str,
    lowpass_length: int,
    highpass_length: int,
) -> dict[str, object]:
    """Apply Pierre's REAL conversion sequence and optional median filters."""
    profiles = np.asarray(raw["profiles"]).sum(axis=1).astype(np.float64, copy=True)
    distances = np.asarray(raw["distances"], dtype=float).copy()
    azimuths = np.asarray(raw["azimuths"], dtype=float)
    elevations = np.asarray(raw["elevations"], dtype=float)

    label = scan_label(azimuths, elevations)

    zero_indices = np.flatnonzero(distances >= 0.0)
    if zero_indices.size == 0:
        raise ValueError("The BSCAN contains no nonnegative range bins.")
    zero_index = int(zero_indices[0])
    background_end = max(zero_index - 1, 1)
    background = profiles[:, :background_end]
    profiles = profiles[:, zero_index:]
    distances = distances[zero_index:]

    # These operations match ScanPreprocessor in Pierre's lidar toolbox.
    profiles -= background.mean(axis=1)[:, np.newaxis]
    distances_km = distances * 1.0e-3
    profiles *= np.square(distances_km)[np.newaxis, :]
    profiles += DEFAULT_DB_OFFSET
    np.clip(profiles, DEFAULT_DB_EPSILON, None, out=profiles)
    np.log10(profiles, out=profiles)
    profiles *= 10.0

    if processing == "filtered":
        profiles = median_filter(
            profiles,
            size=(1, lowpass_length),
            mode="constant",
            cval=0,
        )
        profiles = profiles - median_filter(
            profiles,
            size=(1, highpass_length),
            mode="constant",
            cval=0,
        )
        edge = highpass_length // 2
        profiles = profiles[:, edge:-edge]
        distances = distances[edge:-edge]

    if label == "PPI":
        x, y = polar_grid(distances, elevations * 0 + azimuths, meteo=True)
    elif label == "RHI":
        x, y = polar_grid(distances, elevations, meteo=False)
    else:
        x = y = None

    return {
        "x": x,
        "y": y,
        "scan": profiles,
        "range": distances,
        "azimuth": azimuths,
        "elevation": elevations,
        "time": raw["timestamps"],
        "label": label,
        "filtered": processing == "filtered",
    }


def apply_web_style(cmap: str) -> dict[str, object]:
    """Return Pierre's web style with a caller-selected colormap."""
    return {
        "figure.dpi": 100,
        "figure.figsize": (7.20, 7.20),
        "figure.subplot.left": 0.15,
        "figure.subplot.right": 0.95,
        "figure.subplot.bottom": 0.08,
        "figure.subplot.top": 0.88,
        "axes.facecolor": "0.8",
        "axes.linewidth": 0.8,
        "axes.labelsize": "medium",
        "axes.titlesize": "large",
        "xtick.labelsize": "medium",
        "ytick.labelsize": "medium",
        "xtick.minor.visible": True,
        "ytick.minor.visible": True,
        "grid.color": "white",
        "grid.linestyle": "--",
        "grid.linewidth": 0.8,
        "image.cmap": cmap,
        "pcolor.shading": "nearest",
        "savefig.dpi": 300,
    }


def nice_limit(value: float, step: float, lower: bool) -> float:
    quotient = value / step
    return step * (np.floor(quotient) if lower else np.ceil(quotient))


def plot_scan(
    scan: dict[str, object],
    *,
    vmin: float,
    vmax: float,
    xmin: float | None,
    xmax: float | None,
    ymin: float | None,
    ymax: float | None,
    tick_step: float,
    cmap: str,
    timezone_name: str,
    title_prefix: str,
    array_marker: bool,
    grid: bool,
) -> tuple[plt.Figure, plt.Axes]:
    with plt.rc_context(apply_web_style(cmap)):
        fig, ax = plt.subplots()
        x = scan["x"]
        y = scan["y"]
        if x is None or y is None:
            raise ValueError(f"Cannot display {scan['label']} scans as an x/y image.")
        # Pierre's plot uses a polar-to-Cartesian curvilinear grid. Current
        # Matplotlib versions warn that these cell centers are not monotonic,
        # although this is the intended representation for the scan.
        with warnings.catch_warnings():
            warnings.filterwarnings(
                "ignore",
                message="The input coordinates to pcolormesh are interpreted as cell centers.*",
            )
            image = ax.pcolormesh(
                x, y, scan["scan"], vmin=vmin, vmax=vmax, cmap=cmap, shading="nearest"
            )
        colorbar = fig.colorbar(image, shrink=0.66)

        ax.set_aspect("equal")
        ax.set_xlabel("x (m)")
        ax.set_ylabel("y (m)" if scan["label"] == "PPI" else "z (m)")
        ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_step))
        ax.yaxis.set_major_locator(ticker.MultipleLocator(tick_step))
        ax.xaxis.set_minor_locator(ticker.MultipleLocator(tick_step / 2.0))
        ax.yaxis.set_minor_locator(ticker.MultipleLocator(tick_step / 2.0))

        x_auto_min, x_auto_max = ax.get_xlim()
        y_auto_min, y_auto_max = ax.get_ylim()
        ax.set_xlim(
            x_auto_min if xmin is None else xmin,
            x_auto_max if xmax is None else xmax,
        )
        ax.set_ylim(
            y_auto_min if ymin is None else ymin,
            y_auto_max if ymax is None else ymax,
        )

        timestamp = scan["time"][0]
        utc_text = timestamp.strftime("%Y %b %d - %H:%M:%S UTC")
        try:
            local = timestamp.replace(tzinfo=dt.timezone.utc).astimezone(
                ZoneInfo(timezone_name)
            )
            local_text = f"{local:%Y %b %d - %H:%M:%S} {timezone_name}"
            date_text = f"{utc_text} ({local_text})"
        except ZoneInfoNotFoundError:
            date_text = utc_text

        filter_text = "High-pass median filtered" if scan["filtered"] else "Unfiltered (no median filters)"
        if scan["label"] == "PPI":
            scan_angle = f"elevation={scan['elevation'][0]:.2f}$^\\circ$"
        else:
            scan_angle = f"azimuth={scan['azimuth'][0]:.2f}$^\\circ$"
        ax.set_title(f"{title_prefix}{scan['label']} scan - {scan_angle}\n{date_text}")
        colorbar.ax.set_ylabel(f"Relative backscatter intensity (dB)\n{filter_text}")

        if grid:
            ax.grid()
        if array_marker and scan["label"] == "PPI":
            sonic_x, sonic_y = polar_grid(
                np.array([1180.0, 1425.0]), np.array([78.98]), meteo=True
            )
            ax.plot(sonic_x[0], sonic_y[0], "--", color="w", linewidth=2)

        return fig, ax


def get_arguments(argv: list[str] | None = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Display a REAL M²HATS BSCAN selected by nearest UTC timestamp."
    )
    source = parser.add_mutually_exclusive_group()
    source.add_argument(
        "--base-url",
        default=DEFAULT_BASE_URL,
        help=f"public BSCAN base URL (default: {DEFAULT_BASE_URL})",
    )
    source.add_argument(
        "--local-root",
        type=Path,
        help="local BSCAN root, e.g. /raid/bulk_storage/M2HATS/REAL/bscans",
    )
    source.add_argument(
        "--file",
        type=str,
        help="read one local path or direct HTTP URL instead of selecting by time",
    )
    parser.add_argument("--date", type=parse_date, default=DEFAULT_DATE, help="UTC date (YYYY-MM-DD)")
    parser.add_argument("--time", type=parse_time, default=DEFAULT_TIME, help="UTC time (HH:MM:SS)")
    parser.add_argument("--timeout", type=float, default=30.0, help="network timeout in seconds")
    processing = parser.add_mutually_exclusive_group()
    processing.add_argument("--filtered", dest="processing", action="store_const", const="filtered")
    processing.add_argument("--unfiltered", dest="processing", action="store_const", const="unfiltered")
    parser.set_defaults(processing="filtered")
    parser.add_argument("--lowpass-length", type=parse_odd_length, default=DEFAULT_LOW_PASS)
    parser.add_argument("--highpass-length", type=parse_positive_odd_highpass, default=DEFAULT_HIGH_PASS)
    parser.add_argument("--xmin", type=float, default=None)
    parser.add_argument("--xmax", type=float, default=None)
    parser.add_argument("--ymin", type=float, default=None)
    parser.add_argument("--ymax", type=float, default=None)
    parser.add_argument(
        "--vmin", type=float, default=None,
        help="color-bar minimum in dB; defaults to -1 filtered or 26 unfiltered",
    )
    parser.add_argument(
        "--vmax", type=float, default=None,
        help="color-bar maximum in dB; defaults to 3 filtered or 40 unfiltered",
    )
    parser.add_argument("--cmap", default=DEFAULT_CMAP, help="Matplotlib colormap")
    parser.add_argument("--tick-step", type=float, default=DEFAULT_TICK_STEP, help="major tick spacing in meters")
    parser.add_argument("--timezone", default="America/Los_Angeles", help="display timezone; data remain UTC")
    parser.add_argument("--title-prefix", default=DEFAULT_TITLE)
    parser.add_argument("--scan-index", type=int, default=0, help="scan index within a BSCAN file")
    parser.add_argument("--save", type=Path, help="also save a high-resolution PNG")
    parser.add_argument("--dpi", type=int, default=300, help="PNG resolution for --save")
    parser.add_argument("--no-array-marker", action="store_true", help="omit the M²HATS sonic-array marker")
    parser.add_argument("--grid", action="store_true", help="draw a coordinate grid")
    parser.add_argument("--no-show", action="store_true", help="save without opening an interactive window")
    return parser.parse_args(argv)


def load_requested_file(args: argparse.Namespace) -> tuple[bytes, str, dt.datetime]:
    target = dt.datetime.combine(args.date, args.time)
    if args.file:
        if args.file.startswith(("http://", "https://")):
            parsed = parse_filename(args.file)
            if parsed is None:
                raise RuntimeError("The direct URL does not end in a recognized REAL BSCAN filename.")
            timestamp, name = parsed
            return fetch_url(args.file, args.timeout), args.file, timestamp
        path = Path(args.file).expanduser()
        parsed = parse_filename(path.name)
        if parsed is None:
            raise RuntimeError("The local file does not have a recognized REAL BSCAN filename.")
        timestamp, _ = parsed
        return path.read_bytes(), str(path), timestamp

    days = candidate_days(target)
    if args.local_root:
        candidates = list_local_candidates(args.local_root, days)
    else:
        candidates = list_remote_candidates(args.base_url, days, args.timeout)
    timestamp, name, location = select_nearest(candidates, target)
    if isinstance(location, Path):
        return location.read_bytes(), str(location), timestamp
    return fetch_url(str(location), args.timeout), str(location), timestamp


def main(argv: list[str] | None = None) -> int:
    args = get_arguments(argv)
    if args.tick_step <= 0:
        raise SystemExit("--tick-step must be positive")
    if args.dpi <= 0:
        raise SystemExit("--dpi must be positive")

    data, source, selected_timestamp = load_requested_file(args)
    reader = RealBscanReader()
    raw = reader.read_scan(data, scan_index=args.scan_index)
    scan = preprocess(
        raw,
        processing=args.processing,
        lowpass_length=args.lowpass_length,
        highpass_length=args.highpass_length,
    )
    if args.processing == "filtered":
        default_vmin, default_vmax = DEFAULT_FILTERED_VMIN, DEFAULT_FILTERED_VMAX
    else:
        default_vmin, default_vmax = DEFAULT_UNFILTERED_VMIN, DEFAULT_UNFILTERED_VMAX
    vmin = default_vmin if args.vmin is None else args.vmin
    vmax = default_vmax if args.vmax is None else args.vmax
    if vmin >= vmax:
        raise SystemExit("--vmin must be less than --vmax")
    if scan["label"] not in ("PPI", "RHI"):
        raise SystemExit(f"The selected file contains a {scan['label']} scan; PPI/RHI images are required.")

    fig, _ = plot_scan(
        scan,
        vmin=vmin,
        vmax=vmax,
        xmin=args.xmin,
        xmax=args.xmax,
        ymin=args.ymin,
        ymax=args.ymax,
        tick_step=args.tick_step,
        cmap=args.cmap,
        timezone_name=args.timezone,
        title_prefix=args.title_prefix,
        array_marker=not args.no_array_marker,
        grid=args.grid,
    )

    if args.save:
        args.save.parent.mkdir(parents=True, exist_ok=True)
        fig.savefig(args.save, format="png", dpi=args.dpi)

    selected_delta = abs(selected_timestamp - dt.datetime.combine(args.date, args.time))
    print(f"Selected file: {source}")
    print(f"Selected filename time: {selected_timestamp:%Y-%m-%d %H:%M:%S} UTC")
    print(f"Difference from requested time: {selected_delta}")
    print(f"Scan type: {scan['label']}")
    print(f"Processing: {args.processing}")
    if args.save:
        print(f"Saved PNG: {args.save}")

    if args.no_show:
        plt.close(fig)
    else:
        plt.show()
    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except (HTTPError, URLError, TimeoutError, OSError, ValueError) as exc:
        print(f"Error: {exc}", file=sys.stderr)
        raise SystemExit(1)
