#!/usr/bin/env python3
"""Count the fee settings that Polymarket's public API reports for open markets.

Python 3.10+, standard library only. No credentials, account access or orders.
python3 polymarket_fee_survey.py --markets 1000

Reads open markets from Gamma in pages of 100, ordered by 24-hour volume, and
groups them by the fee fields on each market. One paced pass, no retry.
"""
import argparse
import csv
import io
import json
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen

PAGE = 100
MAX_MARKETS = 2000
MAX_BYTES = 16 * 1024 * 1024
GAMMA = "https://gamma-api.polymarket.com/markets"


def now():
    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()


def fetch_page(offset):
    query = urlencode({"limit": PAGE, "offset": offset, "active": "true", "closed": "false",
                       "order": "volume24hr", "ascending": "false"})
    request = Request(f"{GAMMA}?{query}", headers={"User-Agent": "Vultax-Public-Research-Example/1.1", "Accept": "application/json"})
    with urlopen(request, timeout=30) as response:
        body = response.read(MAX_BYTES + 1)
    if len(body) > MAX_BYTES:
        raise ValueError("Response exceeds this example's 16 MiB limit")
    rows = json.loads(body)
    if not isinstance(rows, list):
        raise ValueError("Expected a JSON array of markets")
    return rows


def fee_group(market):
    """The fee setting exactly as reported. A missing field stays None; it is not read as zero."""
    schedule = market.get("feeSchedule") if isinstance(market.get("feeSchedule"), dict) else {}
    return (market.get("feesEnabled"), market.get("feeType"), schedule.get("rate"),
            schedule.get("exponent"), schedule.get("takerOnly"), schedule.get("rebateRate"))


def charge(group):
    enabled, _fee_type, rate, _exponent, _taker_only, _rebate = group
    if enabled is not True:
        return "no fee: fees not enabled"
    if rate is None:
        return "unknown: enabled without a rate"
    return "no fee: rate 0" if float(rate) == 0 else f"taker rate {float(rate):g}"


def survey(limit, pause):
    markets, seen = [], set()
    started = now()
    # A few extra pages replace markets that repeat while the ranking moves.
    for offset in range(0, limit + 3 * PAGE, PAGE):
        if len(markets) >= limit:
            break
        rows = fetch_page(offset)
        for market in rows:
            key = str(market.get("id"))
            if key not in seen:  # Paging a moving ranking can repeat a market.
                seen.add(key)
                markets.append(market)
        if len(rows) < PAGE:
            break
        time.sleep(pause)
    markets = markets[:limit]
    groups = Counter(fee_group(market) for market in markets)
    examples, listed = {}, {}
    for market in markets:
        group = fee_group(market)
        examples.setdefault(group, market.get("slug"))
        day = str(market.get("createdAt") or "")[:10]
        if day:  # When each group's markets were listed; a missing date is left out, not guessed.
            first, last = listed.get(group, (day, day))
            listed[group] = (min(first, day), max(last, day))
    rows = [{"feesEnabled": g[0], "feeType": g[1], "rate": g[2], "exponent": g[3], "takerOnly": g[4],
             "rebateRate": g[5], "charge": charge(g), "markets": count, "exampleSlug": examples[g],
             "listedFrom": listed.get(g, (None, None))[0], "listedTo": listed.get(g, (None, None))[1]}
            for g, count in groups.most_common()]
    by_charge = Counter()
    for row in rows:
        by_charge[row["charge"]] += row["markets"]
    return {"schemaVersion": "vultax-polymarket-fee-survey-v1", "isLive": False,
            "retrievalStartedAt": started, "retrievalFinishedAt": now(),
            "source": GAMMA, "population": f"{len(markets)} open markets with the highest 24-hour volume at retrieval",
            "marketsRead": len(markets), "groups": rows,
            "byCharge": dict(by_charge.most_common()),
            "limitations": "One pass over a ranking that moves while it is read. Fields are reported by the API, "
                           "not confirmed against executed trades. Markets outside the ranking are not described. "
                           "Settings can change after retrieval."}


def render_csv(result):
    output = io.StringIO(newline="")
    writer = csv.writer(output)
    columns = ["charge", "feeType", "rate", "exponent", "takerOnly", "rebateRate", "feesEnabled", "markets", "listedFrom", "listedTo", "exampleSlug"]
    writer.writerow(columns)
    for row in result["groups"]:
        writer.writerow(["" if row[name] is None else row[name] for name in columns])
    return output.getvalue()


def main():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--markets", type=int, default=1000, help=f"How many open markets to read, 100 to {MAX_MARKETS} (default 1000)")
    parser.add_argument("--pause", type=float, default=0.5, help="Seconds between pages (default 0.5, minimum 0.2)")
    parser.add_argument("--output", type=Path, help="New output directory; an existing one is not overwritten")
    args = parser.parse_args()
    if not PAGE <= args.markets <= MAX_MARKETS:
        parser.error(f"--markets must be between {PAGE} and {MAX_MARKETS}")
    if args.output and args.output.exists():
        parser.error("Output directory already exists; choose a new path")
    try:
        result = survey(args.markets, max(args.pause, 0.2))
    except HTTPError as error:
        retry = error.headers.get("Retry-After")
        sys.exit(f"error: HTTP {error.code} from Gamma{f'; Retry-After: {retry}' if retry else ''}. Nothing was saved; no automatic retry.")
    except (URLError, TimeoutError, ValueError) as error:
        sys.exit(f"error: {error}. Nothing was saved.")
    for row in result["groups"]:
        print(f"{row['markets']:>5}  {row['charge']:<28} {row['feeType'] or '-':<22} e.g. {row['exampleSlug']}")
    print(f"{result['marketsRead']} markets read, finished {result['retrievalFinishedAt']}")
    if args.output:
        args.output.mkdir(parents=True, exist_ok=False)
        (args.output / "fee-survey.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n", encoding="utf-8")
        (args.output / "fee-survey.csv").write_text(render_csv(result), encoding="utf-8", newline="")
        print(f"Saved fee-survey.json and fee-survey.csv to {args.output}")


if __name__ == "__main__":
    main()