From 283716a29a1953b864a6e6d941def5780c181fca Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 14 Aug 2026 21:07:06 +0100 Subject: [PATCH 1/3] feat: Velotrade support, high-level utils, and live-verified tests Transport/auth: - get_auth_headers() on all auth handlers; transport builds request/WS headers through the handler (Authorization: DXAPI + X-Auth-Token), fixing authenticated REST against Velotrade - spec-aligned REST methods: get_users, get_account_metrics/portfolio/ positions/orders/orders_history, query_instruments, get_market_data, place_order, cancel_order, ping, logout, _encode_account - Push fixes: portfolio subscription payload (requestType LIST + accounts), envelope timestamps, send_message no longer recv()s, ping stats setdefault - wait_for_channel(); WS URL builders no longer inject /ws; env_config reads DXTRADE_WS_MARKET_DATA_URL / DXTRADE_WS_PORTFOLIO_URL - declare undeclared httpx dependency Utils and examples: - dxtrade.utils: stream_quotes, open_position (optional stop loss), close_position, flatten, account_is_flat, resolve_account/symbol, order_code - examples/stream_quotes.py and examples/trade_smoke.py Tests and docs: - unit tests for auth headers, transport methods, utils; live Velotrade REST + WebSocket tests; diagnostic scripts (_diag_push, _diag_trade) - AGENTS.md (init), TEST_PLAN_VELOTRADE.md, tests.md with live results - .gitignore: stop ignoring test_*.py test files --- .gitignore | 2 - AGENTS.md | 239 ++++++++++++ CHANGELOG.md | 38 ++ TEST_PLAN_VELOTRADE.md | 300 +++++++++++++++ examples/stream_quotes.py | 64 ++++ examples/trade_smoke.py | 117 ++++++ pyproject.toml | 1 + src/dxtrade/auth.py | 160 ++++++-- src/dxtrade/config.py | 96 ++--- src/dxtrade/env_config.py | 7 + src/dxtrade/transport.py | 625 ++++++++++++++++++++++---------- src/dxtrade/utils.py | 609 +++++++++++++++++++++++++++++++ tests.md | 212 +++++++++++ tests/_diag_push.py | 198 ++++++++++ tests/_diag_trade.py | 317 ++++++++++++++++ tests/test_auth.py | 63 +++- tests/test_transport.py | 466 ++++++++++++++++++++++++ tests/test_utils.py | 267 ++++++++++++++ tests/test_velotrade_live.py | 126 +++++++ tests/test_velotrade_ws_live.py | 275 ++++++++++++++ 20 files changed, 3905 insertions(+), 277 deletions(-) create mode 100644 AGENTS.md create mode 100644 TEST_PLAN_VELOTRADE.md create mode 100644 examples/stream_quotes.py create mode 100644 examples/trade_smoke.py create mode 100644 src/dxtrade/utils.py create mode 100644 tests.md create mode 100644 tests/_diag_push.py create mode 100644 tests/_diag_trade.py create mode 100644 tests/test_transport.py create mode 100644 tests/test_utils.py create mode 100644 tests/test_velotrade_live.py create mode 100644 tests/test_velotrade_ws_live.py diff --git a/.gitignore b/.gitignore index 47f138e..8d37bf0 100644 --- a/.gitignore +++ b/.gitignore @@ -156,8 +156,6 @@ test-outputs/ CLAUDE.md # Test files -test_*.py -*_test.py CONFIRMATION.md # Development files diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..959a3c6 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,239 @@ +# AGENTS.md + +Guidance for AI coding agents working in this repository. Read this before making changes. + +## Project Overview + +`dxtrade-sdk` is a Python SDK for the **DXTrade** trading platform (a white-label multi-asset broker platform used by many brokers). It is an unofficial SDK with two layers: + +- **Transport layer** — `dxtrade.transport.DXTradeTransport` (factory: `create_transport()`). A minimal, raw-data-passthrough client designed for building bridges and middleware (RabbitMQ, Kafka, Redis). It handles session-token authentication, raw REST requests with auth headers, multi-channel WebSocket subscriptions with raw message forwarding, and automatic application-level ping/pong session extension. No data modeling — returns raw JSON. +- **High-level SDK layer** — `dxtrade.client.DXTradeClient` (factories: `create_client`, `create_demo_client`, `create_live_client`), typed REST modules (`dxtrade.rest`), WebSocket stream managers (`dxtrade.websocket`), and Pydantic type models (`dxtrade.types`, `dxtrade.models`). This layer mirrors the structure of the official TypeScript SDK. + +The project is **platform-agnostic**: it works with any DXTrade broker purely through environment configuration (`DXTRADE_*` variables), with no broker-specific code hardcoded. + +- Python `>=3.10`, fully async (`asyncio`), MIT licensed. +- Version `1.0.0` (matches `__version__` in `src/dxtrade/__init__.py` and `pyproject.toml`). +- Package name on PyPI: `dxtrade-sdk`. Public entry points are `dxtrade.create_transport`, `dxtrade.DXTradeTransport`, and `dxtrade.__version__` (see `src/dxtrade/__init__.py`). + +## Repository Layout + +``` +pyproject.toml # Build (hatchling), lint, type-check, test config +setup.py # Backwards-compat shim; real config lives in pyproject.toml +README.md # Primary documentation (usage, config, publishing) +CHANGELOG.md # Keep a Changelog + SemVer +LICENSE # MIT +MANIFEST.in # sdist include rules +.env.example # Template for broker credentials / config (.env is gitignored) +.github/workflows/ci.yml # CI: test matrix, lint, mypy, security, docs + +src/dxtrade/ + __init__.py # Public API surface (transport only) + transport.py # DXTradeTransport — raw REST + WebSocket passthrough (primary layer) + auth.py # Auth handlers: BearerTokenHandler, HMACHandler, SessionHandler, AuthFactory + utils.py # High-level helpers on the transport: stream_quotes, open/close_position, flatten + models.py # Pydantic models (credentials, accounts, orders, positions, events) + errors.py # DXtrade* exception hierarchy + config.py # Dataclass-based SDKConfig + sub-configs (Endpoints, WebSocketConfig, ...) + env_config.py # Load SDKConfig from DXTRADE_* environment variables + client.py # DXTradeClient + factory functions + core/http_client.py # aiohttp-based HTTPClient (rate limiting, retries) + rest/ # AccountsAPI, OrdersAPI, PositionsAPI, InstrumentsAPI + websocket/ # DXTradeStreamManager, UnifiedWebSocketStream (dual-connection) + types/ # Pydantic type models mirroring the TypeScript SDK + common.py # Environment, auth configs, SDKConfig, ApiResponse, ... + trading.py # Account, Instrument, Order, Position, Quote, ... + websocket.py # Generic WS message models and callbacks + dxtrade_messages.py # DXTrade-specific WS messages (Ping, MarketData, ...) + +examples/ + stream_market_data.py # Authenticate + stream quotes via transport layer + stream_quotes.py # Stream quotes via dxtrade.utils.stream_quotes (--symbols/--duration) + trade_smoke.py # Open/close a small position via utils (--stop-loss, --dry-run) + bridge_example.py # Bridge pattern: forward WS data to a message queue + README.md # Example docs + +tests/ + conftest.py # Fixtures (credentials, models, httpx mocks) + test_auth.py # Auth handler unit tests (bearer, HMAC, session, factory) +``` + +There is no `docs/` directory yet, although `pyproject.toml`, `MANIFEST.in`, and CI reference one (`docs/` sdist include, `mkdocs build` step, `docs/MIGRATION.md` in `CHANGELOG.md`). Do not assume it exists. + +## Two Configuration Systems + +The codebase contains **two parallel, incompatible config systems** — keep this in mind before touching either: + +1. **Dataclass-based** (`config.py` + `env_config.py`) — `SDKConfig` (alias `DXTradeConfig`) with `AuthConfig`, `Features`, `Endpoints`, `WebSocketConfig`, `RateLimitConfig`, `RetryConfig`. Loaded from environment via `load_config_from_env()`. **This is what the transport layer uses.** +2. **Pydantic-based** (`types/common.py`) — pydantic `SDKConfig` with `AuthConfig` as a Union of `SessionAuth` / `BearerAuth` / `HmacAuth` / `CredentialsAuth`, plus `RateLimitConfig`, `FeaturesConfig`, `URLsConfig`, `EndpointsConfig`, `WebSocketConfig`. **This is what the high-level SDK layer (`client.py`, `core/`, `rest/`, `websocket/`) uses.** + +The two `AuthConfig`, `SDKConfig`, and `WebSocketConfig` names are different classes with different shapes — do not assume you can pass one where the other is expected. The high-level layer also expects auth as a Pydantic model with a `type` field (`"session"`, `"bearer"`, `"hmac"`, `"credentials"`), while the dataclass layer uses the `AuthType` enum defined in `config.py` (a third, separate `AuthType` enum also exists in `models.py` with values `bearer_token`/`hmac`/`session`). + +## Environment Configuration + +Configuration comes from a `.env` file (loaded via `python-dotenv` by the transport) or environment variables. All variables use the `DXTRADE_` prefix. See `.env.example` for the full annotated template. + +Key variables: + +| Variable | Purpose | +|---|---| +| `DXTRADE_USERNAME`, `DXTRADE_PASSWORD`, `DXTRADE_DOMAIN` | Credentials auth (domain defaults to `default`) | +| `DXTRADE_SESSION_TOKEN` | Session-token auth (alternative to credentials) | +| `DXTRADE_BEARER_TOKEN` / `DXTRADE_API_KEY` + `DXTRADE_API_SECRET` | Bearer / HMAC auth | +| `DXTRADE_BASE_URL` | REST base URL, e.g. `https://your-broker.com/dxsca-web` | +| `DXTRADE_WS_MARKET_DATA_URL` | Market data WS URL, e.g. `wss://.../dxsca-web/md?format=JSON` | +| `DXTRADE_WS_PORTFOLIO_URL` | Portfolio/account WS URL | +| `DXTRADE_ACCOUNT` | Account ID, e.g. `default:demo`; falls back to `default:` | +| `DXTRADE_TIMEOUT` | Request timeout (default 30 s) | +| `DXTRADE_WS_PING_INTERVAL`, `DXTRADE_WS_RECONNECT_ATTEMPTS`, `DXTRADE_WS_RECONNECT_DELAY` | WebSocket tuning | +| `DXTRADE_LOG_LEVEL` | `DEBUG`/`INFO`/`WARNING`/`ERROR` | +| `DXTRADE_ENDPOINT_*`, `DXTRADE_WS_*`, `DXTRADE_RATE_LIMIT_*`, `DXTRADE_RETRY_*`, `DXTRADE_FEATURE_*` | Endpoint paths, WS paths/format, rate limiting, retry, feature flags | + +`env_config.py` documents every supported variable in its `load_config_from_env()` docstring. Note a quirk: `DXTRADE_WS_MARKET_DATA_URL`/`DXTRADE_WS_PORTFOLIO_URL` are used by the transport layer, while `env_config.py` itself reads `DXTRADE_WS_URL` + `DXTRADE_WS_MARKET_DATA_PATH`/`DXTRADE_WS_PORTFOLIO_PATH` — the transport's `subscribe()` uses the explicit URL variables when present. + +## Build and Test Commands + +Development environment: + +```bash +python -m venv venv +source venv/bin/activate # Windows: venv\Scripts\activate +pip install -e ".[dev]" +``` + +Tests (pytest — config in `[tool.pytest.ini_options]`): + +```bash +pytest # runs tests/, asyncio_mode=auto +pytest tests/test_auth.py # single file +pytest --cov=dxtrade # coverage report (term-missing) +``` + +`pyproject.toml` sets coverage enforcement: `--cov=dxtrade`, `--cov-fail-under=90` (branch coverage on `src`). **Warning:** in the current tree the 24 tests pass, but total coverage is only ~16%, so a bare `pytest` exits with code 1 (coverage gate fails). See "Known Issues" below. + +Code quality (all configured in `pyproject.toml`, all run in CI): + +```bash +ruff check src tests # lint (E/W/F/I/B/C4/UP/RUF; E501, B008, C901 ignored) +black --check src tests # formatting (line-length 88) +mypy src # strict typing +``` + +Fix formatting with `black src tests` and auto-fix lint with `ruff check src tests --fix`. + +Build and publish (from README): + +```bash +python -m build # build sdist + wheel +python -m twine upload --repository testpypi dist/* # test first +python -m twine upload dist/* # then publish +``` + +Bump the version in **both** `pyproject.toml` and `src/dxtrade/__init__.py`, and add a `CHANGELOG.md` entry. Follow SemVer. + +## Testing Strategy + +- **Test framework:** pytest with `pytest-asyncio` in `asyncio_mode = "auto"` (async tests need no explicit marker). Markers registered: `unit`, `integration`, `slow` (there are currently no integration/slow tests). +- **Current coverage:** only `tests/test_auth.py` exists (auth handler unit tests). Tests mock `httpx.AsyncClient`/`httpx.Request` via `unittest.mock` fixtures in `tests/conftest.py` — they never touch the network. Verified state: **24 tests, all passing**, but ~16% total coverage, so the configured `--cov-fail-under=90` gate fails (pytest exit code 1). Raising coverage to ≥90% is open work, not done work. +- **Pattern to follow:** per-class test classes (`TestBearerTokenHandler`, `TestSessionHandler`, ...), fixtures in `conftest.py`, descriptive test names (`test_authenticate_login_failure`), and assertions on both behavior and call arguments (`client.post.assert_called_once_with(...)`). +- `tests/conftest.py` requires `httpx` at import time, and `tests/test_auth.py` imports `dxtrade.auth`, which imports `httpx` and `dxtrade.models` — the test suite depends on the `dxtrade` package being importable. + +## Code Style Guidelines + +- **Line length 88** (black default; ruff `E501` ignored because black handles it). Black-formatted, ruff-linted (see config above). +- **Strict typing:** mypy runs in `strict` mode on `src` (`disallow_untyped_defs`, `warn_return_any`, `no_implicit_optional`, `extra_checks`, ...). Tests are exempt from `disallow_untyped_defs`. All public functions are fully typed. +- **Type hints:** modern syntax (`Optional[str]`, `Dict[str, Any]` from `typing`; `str | None` is *not* used). Pydantic models use `Field(..., description=...)` everywhere. +- **Docstrings:** Google-style with `Args:` / `Returns:` / `Raises:` sections on every public class/method. +- **Logging:** use `logging.getLogger(__name__)` per module, and `logger.info/debug/warning/error` — not `print` (a few `print` calls remain in `websocket/stream_manager.py`; do not extend that pattern). +- **Emoji in log/print messages:** the codebase liberally uses emoji in user-facing log lines and example output (e.g. `✅`, `❌`, `🔌`, `📡`, `📤`, `🔄`, `📈`). Match this convention in new transport/example code. +- **Enum conventions:** `models.py` uses lowercase enum values (`"buy"`, `"market"`, `"open"`); `types/trading.py` uses uppercase (`"BUY"`, `"MARKET"`). `types/common.py` `Environment` uses lowercase (`"demo"`/`"live"`). Match the file you are editing. +- **Secrets hygiene:** credentials models use `repr=False` on secret fields (`HMACCredentials.secret_key`, `SessionCredentials.password`); `SDKConfig.to_dict()` deliberately omits sensitive auth fields. Do not log tokens or passwords. + +## Architecture Notes + +### Transport layer (`transport.py`) — the working, documented core + +- `DXTradeTransport.authenticate()` POSTs `{"username", "password", "domain"}` to the login endpoint and stores `sessionToken` (expires after 1 hour, refreshed lazily on 401). +- REST: `request(method, endpoint, ...)` attaches the auth handler's headers (for session auth: both `X-Auth-Token` and `Authorization: DXAPI `), parses JSON or text, and auto-refreshes the token once on 401. Spec-aligned helpers exist for the official DXTrade REST API: `get_users`, `get_account_metrics`, `get_account_portfolio`, `get_account_positions`, `get_account_orders`, `get_account_orders_history`, `query_instruments`, `get_market_data`, `ping`, `logout`. +- WebSocket: `subscribe(channel, callback, ws_url)` opens a connection per channel ("quotes"/"market_data" → market data URL, everything else → portfolio URL) and forwards raw parsed messages to the callback. `wait_for_channel(channel, timeout)` waits until the background connection is established. When `ws_url` is omitted the URL comes from the config (`DXTRADE_WS_MARKET_DATA_URL` / `DXTRADE_WS_PORTFOLIO_URL`, else constructed from the base URL as `/md` and `/` — no `/ws` segment). +- **4-tier connection fallback** for `websockets` library compatibility: `additional_headers` (v12+) → `extra_headers` (v9–10) → subprotocol auth → post-connection auth message. Tracked per channel in `get_connection_strategies()`. +- **Application-level ping/pong:** server sends `{"type": "PingRequest"}`; the transport replies `{"type": "Ping", "session": , "timestamp": }` and *does not* forward ping messages to user callbacks. Stats via `get_ping_stats()` / `get_session_health()`. +- Subscription messages (sent by `send_market_data_subscription` / `send_portfolio_subscription`) use DXTrade's protocol: `MarketDataSubscriptionRequest` (payload: `account`, `symbols`, `eventTypes: [{"type": "Quote", "format": "COMPACT"}]`) and `AccountPortfoliosSubscriptionRequest` (payload: `requestType: "LIST"`, `accounts: [...]`), each with a `requestId`, `timestamp`, and `session`. +- Incoming data shapes: `{"type": "MarketData", "payload": {"events": [{"symbol", "bid", "ask", "timestamp"}]}}` and `{"type": "AccountPortfolios", "payload": {...}}`. + +### Authentication (`auth.py`) + +- `AuthHandler` ABC with `authenticate(request, client)` and `get_auth_headers()` (transport-agnostic header builder); implementations: `BearerTokenHandler`, `HMACHandler` (signs `timestamp + method + path + body [+ passphrase]` with HMAC-SHA256, headers `DX-API-KEY`, `DX-API-TIMESTAMP`, `DX-API-SIGNATURE`, `DX-API-PASSPHRASE`), `SessionHandler` (sends **both** `X-Auth-Token` and `Authorization: DXAPI ` headers; token auto-refresh, 1 h expiry with 5 min buffer, `logout()`). +- The transport layer builds request/WS headers through `auth_handler.get_auth_headers()`, so each broker's auth scheme (header names, token formats) lives in its handler rather than being hardcoded in the transport. +- `AuthFactory.create_handler(auth_type, credentials)` with `register_handler` for custom handlers. `AuthType` enum lives in `models.py` (`bearer_token`/`hmac`/`session`). + +### High-level utilities (`utils.py`) + +Convenience wrappers over the transport for the common flows — account discovery, +symbol resolution, streaming, and order lifecycle. All helpers work with any +DXTrade broker: + +- `resolve_account(transport)` / `find_account_code(users)` — authenticate on + demand and extract the account code (`default:12345`) from `/users`. +- `resolve_symbol(transport, account, hint)` — resolve user hints to platform + symbols (e.g. `BTCUSDT` → `BTCUSD`); `discover_symbols()` returns a few + tradable symbols. +- `stream_quotes(transport, symbols, duration, on_quote)` — subscribe, collect + quote events for a duration, unsubscribe; returns the events list. +- `open_position(transport, symbol, side, quantity, stop_loss, stop_loss_price, + account)` — market order with optional protective stop. `stop_loss` is a max + loss in account currency (converted to a price from the quote); + `stop_loss_price` is an absolute price. The protective stop is placed as a + closing STOP order **without a quantity** (the DXTrade API rejects closing + STOP/LIMIT orders that carry one — `errorCode 33`). +- `close_position(transport, symbol, position_code, account)` — market close of + exactly one matching position; `flatten(transport)` closes everything and + cancels working orders; `account_is_flat(transport)` checks + `openPositionsCount`/`openOrdersCount` via `/metrics`. +- `order_code(prefix)` — unique client order codes (required per account). + +Note: closing orders and the utils' stop placement omit `quantity` (full close). +Protective stops auto-cancel on the platform when their position closes. + +### High-level SDK layer (broken in the current tree — see Known Issues) + +`client.py`, `core/`, `rest/`, `websocket/` implement the typed client: `DXTradeClient` exposes `accounts`/`orders`/`positions`/`instruments` REST modules and `create_stream` / `start_stream` / `create_unified_stream` WebSocket entry points. The WebSocket managers (`websocket/stream_manager.py`) implement the dual-connection (market data + portfolio) architecture from the TypeScript SDK, with auto-reconnect, ping/pong, and a `run_stability_test()`. + +## Known Issues (as of the current tree) + +Verified against a fresh venv with runtime deps installed (import checks, `pytest`, `ruff`, `black --check`, `mypy` all executed on this tree) — treat the high-level layer with caution: + +- **The high-level SDK layer does not import cleanly.** Several modules reference names that do not exist, which raises `ImportError` at import time (confirmed): + - `client.py` imports `ConfigError` from `dxtrade.errors` (only `DXtradeConfigurationError` exists) and references an undefined `HttpClient`. + - `rest/*.py` and `core/http_client.py` fail at import: `core/http_client.py` imports `CredentialsAuth`, `HTTPMethod`, `ApiResponse` from `dxtrade.config` (those live in `types/common.py`, not `config.py`) and calls `config.rate_limit.window`, which the dataclass `RateLimitConfig` does not define. + - `websocket/*.py` import `WebSocketError` from `dxtrade.errors` (exists only as `DXtradeWebSocketError`). + - `core/__init__.py` imports `WebSocketClient` from `core/websocket_client.py`, which does not exist. + - Consequence: `import dxtrade` works (transport layer only), but `import dxtrade.client` / `dxtrade.rest.*` / `dxtrade.websocket.*` / `dxtrade.core` all raise `ImportError`. +- **Undeclared dependency:** `auth.py` imports `httpx` — now declared in `pyproject.toml` dependencies (fixed in Unreleased). +- **All quality gates are currently red** (measured on this tree): + - `pytest`: 24 tests pass, but coverage is 15.9% vs the required 90% → exit code 1. + - `ruff check src tests`: 1591 errors (1329 auto-fixable) — most are formatting (the tree was not run through `black`). + - `black --check src tests`: 25 files would be reformatted. + - `mypy src`: 272 errors in 16 files (strict mode), e.g. untyped functions in `client.py` and `str` vs `Environment` mismatches. + - Consequently CI (`.github/workflows/ci.yml` runs all four) would fail on the current tree. +- **Two `AuthConfig`/`SDKConfig`/`WebSocketConfig` definitions** (dataclass vs pydantic) — see "Two Configuration Systems" above. +- `docs/` is referenced by build/CI but absent. +- The public `__init__.py` deliberately exports only the transport layer — `DXTradeClient` is documented in README/CHANGELOG but not exported from `dxtrade` (the README Quick Start also uses `from dxtrade import create_transport`). + +When fixing these: prefer the transport layer's working patterns, add `httpx` to `pyproject.toml` dependencies, and consolidate the two config systems rather than adding a third. + +## Security Considerations + +- **Never commit credentials.** `.env` is gitignored; only commit the `.env.example` template. Credentials go in environment variables, never hardcoded. +- Session tokens and passwords are marked `repr=False` in Pydantic models — keep it that way. Do not log tokens, headers, or request bodies containing credentials. +- The SDK sends session tokens in both `X-Auth-Token` and `Authorization` headers — both are required by the DXTrade API; do not "simplify" this. +- CI runs `bandit -r src/` and `safety check` (results uploaded as artifacts; failures are non-fatal via `|| true`). +- HMAC signing includes method, path, query, body, and optional passphrase; timestamps prevent replay. +- Trading involves real money — the transport forwards raw data untouched and never fabricates messages; keep it that way. + +## Conventions Summary + +- Follow the existing per-module conventions; match the style of the file you edit (black + ruff + mypy strict are enforced in CI). +- English (US) for all comments, docstrings, and docs. +- Keep the transport layer minimal and raw — that is its stated design goal ("~200 lines vs 2000+ for full SDK"). +- When changing message formats or env vars, update `.env.example`, `README.md`, and `CHANGELOG.md` (SemVer + Keep a Changelog). diff --git a/CHANGELOG.md b/CHANGELOG.md index 90e2685..13b6c4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,44 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- Declared the undeclared `httpx` runtime dependency in `pyproject.toml` +- REST requests now attach the broker's auth headers via the auth handler + (`Authorization: DXAPI ` plus `X-Auth-Token` for session auth), + fixing authenticated requests against DXTrade brokers such as Velotrade +- Portfolio subscription now uses the DXTrade Push API payload shape + (`requestType: "LIST"` + `accounts: [...]`); the previous shape is + rejected by the server (`errorCode 32`) +- `send_message()` no longer reads from the socket (the background message + handler owns `recv`), fixing a `websockets.ConcurrencyError` on send +- Push subscription messages now carry the spec-required `timestamp` field +- Ping stats are stored per channel even before a full connection cycle, + fixing lost counters + +### Added +- `AuthHandler.get_auth_headers()` — transport-agnostic auth header builder + implemented by `SessionHandler`, `BearerTokenHandler`, and `HMACHandler` +- Transport REST methods matching the official DXTrade OpenAPI spec: + `get_users`, `get_account_metrics`, `get_account_portfolio`, + `get_account_positions`, `get_account_orders`, `get_account_orders_history`, + `query_instruments`, `get_market_data`, `place_order`, `cancel_order`, + `ping`, `logout`, and `_encode_account` for percent-encoding account codes +- `dxtrade.utils` — high-level helpers for streaming and trading: + `stream_quotes`, `open_position` (with protective stop loss), + `close_position`, `flatten`, `account_is_flat`, `resolve_account`, + `resolve_symbol`, `discover_symbols`, `order_code` +- `DXTradeTransport.wait_for_channel()` — wait for a WebSocket channel to connect +- Example scripts: `examples/stream_quotes.py`, `examples/trade_smoke.py` +- `env_config.py` now reads the documented `DXTRADE_WS_MARKET_DATA_URL` and + `DXTRADE_WS_PORTFOLIO_URL` variables + +### Fixed +- WebSocket URL fallback builders no longer inject a wrong `/ws` path segment + (`get_market_data_url`/`get_portfolio_url`), and `subscribe()` resolves URLs + through them + ## [1.0.0] - 2025-01-06 ### Added diff --git a/TEST_PLAN_VELOTRADE.md b/TEST_PLAN_VELOTRADE.md new file mode 100644 index 0000000..28299f1 --- /dev/null +++ b/TEST_PLAN_VELOTRADE.md @@ -0,0 +1,300 @@ +# Velotrade Integration Test Plan — `dxtrade-python-sdk` + +Scope: validating `dxtrade-sdk` (transport layer primarily) against Velotrade's DXtrade +platform deployment. Discovery performed live on **2026-08-14** via browser session +(sources listed in §10). All live probes were **unauthenticated** and read-only; every +test that needs credentials is marked with a credential requirement. + +--- + +## 1. Purpose and Objectives + +1. Prove the SDK can authenticate against Velotrade (`POST /dxsca-web/login`) and obtain a `sessionToken`. +2. Prove REST read-only access works (account discovery, metrics, instruments, positions, orders, market data) using Velotrade's auth scheme. +3. Prove the Push (WebSocket) API works: market-data stream, business/portfolio stream, application-level ping/pong session extension. +4. Prove the order lifecycle safely (demo/sandbox only): open, modify, close, error handling. +5. Identify and document every place the SDK's assumptions differ from Velotrade's live API (see §8 "Known gaps"). +6. Prove the SDK never leaks credentials and handles rate limits/backpressure without blind retries. + +**Non-goals:** FIX API (not provisioned by Velotrade), the broken high-level SDK layer +(`dxtrade.client`, `dxtrade.rest`, `dxtrade.websocket`, `dxtrade.core` — known import +failures, tracked separately), HMAC/Bearer auth (Velotrade trading accounts use session +token auth only). + +--- + +## 2. Live-Verified Velotrade Environment (Discovery Output) + +| Item | Value | Verification | +|---|---|---| +| REST base URL | `https://dx.velotrade.com/dxsca-web` | Login probe returned DXTrade JSON errors; Swagger served at `/dxsca-web/api/swagger.html` | +| REST login path | `POST /dxsca-web/login` | Empty body → `500 errorCode 110`; bad creds → `401 errorCode 3 "Authorization failed"` | +| REST login body | `{"username","password","domain"}` | Matches `LoginRequest` schema in OpenAPI | +| REST login domain | `default` | **Not** `vendor=velotrade` (website SSO param ≠ REST domain) | +| REST auth header | `Authorization: DXAPI ` | OpenAPI `securitySchemes.Authorization`; Velotrade docs | +| REST extra requirement | Non-empty `User-Agent` header (else 403 per spec) | REST spec "Authentication" | +| Business Push (WS) | `wss://dx.velotrade.com/dxsca-web/?format=JSON` — **trailing slash required** | Live open; `/dxsca-web/ws` and no-slash variants fail | +| Market-data Push (WS) | `wss://dx.velotrade.com/dxsca-web/md?format=JSON` | Live open (format param optional) | +| Account code format | `default:` e.g. `default:130000505`, URL-encoded `default%3A...` in paths | Velotrade blog §2 | +| Account discovery | `GET /dxsca-web/users` → account records | OpenAPI `/users`; Velotrade blog | +| Push ping protocol | Server `{"type":"PingRequest","session","timestamp"}` → client `{"type":"Ping","session","timestamp"}` | Push API spec §Ping — **identical to SDK `_handle_ping_pong`** | +| Order endpoint | `POST /dxsca-web/accounts/{encodedAccountCode}/orders` | OpenAPI `SingleOrderRequest` | +| Not standalone (404) | `/dxsca-web/instruments`, `/dxsca-web/accounts`, `/dxsca-web/orders`, `/dxsca-web/positions`, `/dxsca-web/quotes`, `/dxsca-web/time` | Live probes | + +### 2.1 REST endpoints present (OpenAPI `openapi.json`, live at `/dxsca-web/swagger/openapi.json`) + +``` +POST /login POST /loginByToken POST /logout POST /ping +GET /users GET /users/{username} +GET /accounts/{account}/events GET /accounts/events +GET /accounts/{account}/metrics GET /accounts/metrics +GET /accounts/{account}/portfolio GET /accounts/portfolio +POST /accounts/{account}/close +POST /accounts/{account}/transfers GET /accounts/transfers +GET /accounts/{account}/orders/history GET /accounts/orders/history +GET /accounts/{account}/instruments/{symbol} +POST /accounts/{account}/instruments/query +GET /accounts/{account}/instruments/type/{type} +GET /instruments/{symbol} POST /instruments/query GET /instruments/type/{type} +POST /marketdata +POST /accounts/{account}/orders POST /accounts/orders +GET /accounts/{account}/orders/{order} +POST /accounts/{account}/orders/group +GET /accounts/{account}/positions GET /accounts/positions +GET /accounts/{account}/tvLoginInfo GET /accounts/tvLoginInfo +GET /accounts/eodmetrics/{date} GET /conversionRates +``` + +### 2.2 Push API message shapes (from Velotrade Push API spec) + +- **Envelope:** `type` (required), `requestId` (≤64 chars), `inReplyTo`, `refRequestId`, + `timestamp` (required, ISO-8601 UTC), `session`, `principal`/`hash` (HMAC only), `payload`. +- **Market data sub:** `MarketDataSubscriptionRequest` → payload `{account, symbols, + eventTypes:[{type:"Quote",format:"COMPACT"}]}` → server `MarketData` with + `payload.events[]` (`{symbol,type,bid,ask,time}`). **Matches SDK exactly** (SDK omits `timestamp`). +- **Portfolio sub:** `AccountPortfoliosSubscriptionRequest` → payload + `{requestType:"LIST", accounts:["default:..."]}` → server `AccountPortfolios` with + `payload.portfolios[]`. **SDK payload differs** — SDK sends `{account, eventTypes:[{type:"Position",format:"COMPACT"}]}`. +- **Close sub:** `AccountPortfoliosCloseSubscriptionRequest` with `refRequestId` → server + `AccountPortfoliosSubscriptionClosed`. SDK has no explicit close message. +- **Errors:** code `1` auth required (missing/expired session), `2` entity not found, + `32` incorrect request parameters, `34` no market-data permission, `429` too many + requests (default 1/min for `RequestType=ALL` subscriptions). WS close `1013` = backpressure. + +### 2.3 REST error table (Velotrade blog "Common Errors") + +| HTTP / code | Meaning | Safe action | +|---|---|---| +| 401 / 3 | Bad credentials, wrong domain, or account lock | Use domain `default`; stop retrying | +| 404 (HTML) | Wrong path shape | Use full encoded account code | +| 400 / 32 | Incorrect parameters | Validate fields/enums/account/symbol | +| 400 / 33 | Malformed/incompatible order | Validate order schema locally | +| 409 / 100 | Duplicate client identifier | Reconcile; do not blindly retry | +| 429 | Rate limit | Back off; never blind-retry a trade | + +### 2.4 Order placement (OpenAPI `SingleOrderRequest`) + +Fields: `account`, `orderCode` (client-generated, unique per account), `metadata`, +`type` (`MARKET|LIMIT|STOP`), `instrument`, `quantity` (base-currency units — forex lots +×100,000), `positionEffect` (`OPEN|CLOSE`), `positionCode`, `side` (`BUY|SELL`), +`limitPrice`, `stopPrice`, `priceOffset`, `priceLink`, `tif` (`DAY|GTC|IOC|FOK|GTD`), +`marginRate`, `expireDate`. A `200` response with an order id is an **acknowledgement, +not execution proof** — confirm via `/accounts/{code}/orders/history` or Push. + +--- + +## 3. Prerequisites and Test Environment + +- **Credentialed account:** one Velotrade challenge/eval account (username/password are + the API credentials). Recommended: a fresh small 1-step or 2-step account with no open + positions for phases A–D; a **demo/sandbox** account for phase E. Never run phase E on a + funded account. +- **Python:** 3.10+; `pip install -e ".[dev]"` in the repo venv. +- **Network access:** `dx.velotrade.com` (REST + WSS). Corporate proxies break WS — verify first. +- **Clock sync:** REST HMAC not used, but Push timestamps should be near server time. +- **Secrets:** credentials only in `.env` (gitignored) or env vars; never in code/logs/CI. +- **Test data conventions:** every order uses a unique `orderCode` (e.g. `vt-`); + every subscription a unique `requestId`. + +### 3.1 Recommended `.env` (place under `tests/velotrade/.env` or env vars — never commit) + +```bash +DXTRADE_BASE_URL=https://dx.velotrade.com/dxsca-web +DXTRADE_USERNAME= +DXTRADE_PASSWORD= +DXTRADE_DOMAIN=default +DXTRADE_ACCOUNT=default: # from GET /users — NOT the portal account id +DXTRADE_USER_AGENT=dxtrade-sdk-velotrade-tests/1.0 +DXTRADE_LOG_LEVEL=INFO +DXTRADE_TIMEOUT=30 +# NOTE: these two are documented by the SDK but NOT currently read by env_config.py +# (see §8 gap G2). Pass ws_url explicitly to subscribe() or patch env_config first. +DXTRADE_WS_MARKET_DATA_URL=wss://dx.velotrade.com/dxsca-web/md?format=JSON +DXTRADE_WS_PORTFOLIO_URL=wss://dx.velotrade.com/dxsca-web/?format=JSON +``` + +--- + +## 4. Test Architecture + +- **Framework:** pytest + pytest-asyncio (`asyncio_mode=auto`), same conventions as `tests/test_auth.py`. +- **Three layers of tests:** + 1. **Mocked/offline** (default `pytest`, no credentials): config loading, auth-header + construction, message-shape builders, ping/pong handling, URL fallback logic — all + against recorded fixtures. + 2. **Contract tests** (no credentials, live): endpoint existence / auth rejection + probes (`401`/`404` shape checks) — safe to run in CI. + 3. **Live integration** (credentials required, `-m live`, opt-in): real login, REST + reads, Push subscriptions, order lifecycle. +- Markers: reuse `unit`; add `live` and `slow`. Keep live tests gated behind a + `--live`/env flag so the suite never hits the network by default. +- Fixtures in `tests/velotrade/conftest.py`: `live_env` (env-file loader), `transport` + (factory + clean shutdown), `session_token` (one login per module, reused), `account_code` + (from `/users`), `instrument` (smallest tradable discovered). + +--- + +## 5. Test Phases and Cases + +### Phase A — Configuration and URL Contract (no credentials, offline + live probes) + +| ID | Pri | Test | Steps | Expected result | +|---|---|---|---|---| +| A1 | High | `.env` loads for Velotrade values | Load `DXTRADE_BASE_URL`, auth, account from env via `load_config_from_env()` | `base_url == https://dx.velotrade.com/dxsca-web`; `auth.type == CREDENTIALS`; `domain == default` | +| A2 | High | Login URL construction | `transport.authenticate()` path resolution | URL = `https://dx.velotrade.com/dxsca-web/login` (no double slash, no `/dxsca-web/login` duplication) | +| A3 | High | MD WS URL precedence | `subscribe("quotes", cb, ws_url=…)` and config fallback | Explicit `ws_url` wins; config `market_data_url` used when set; otherwise `ValueError` (never silent wrong URL) | +| A4 | High | Portfolio WS URL for Velotrade | `subscribe("portfolio", cb, ws_url="wss://dx.velotrade.com/dxsca-web/?format=JSON")` | Connects (see D1). Assert trailing-slash URL, not `/ws` | +| A5 | Med | Endpoint existence probes (unauthenticated) | GET `/dxsca-web/ping`? (POST), `/users`, `/accounts/{code}/metrics` with no/bad auth | `401 errorCode 1` (auth required), **not** 404 — proves path exists | +| A6 | Med | Non-existent SDK defaults | GET `/dxsca-web/orders`, `/positions`, `/quotes`, `/time`, `/instruments` | 404 HTML — documents that SDK convenience methods hit dead paths (gap G6) | +| A7 | Med | Swagger/spec reachability | GET `/dxsca-web/swagger/openapi.json` | 200 JSON, paths superset of §2.1 | + +### Phase B — Authentication and Session Lifecycle (credentials) + +| ID | Pri | Test | Steps | Expected result | +|---|---|---|---|---| +| B1 | High | Successful login | `transport.authenticate()` | Returns non-empty `sessionToken`; stored in handler; expiry ≈ +1 h | +| B2 | High | Wrong password | Login with bad password | `DXtradeAuthenticationError`; HTTP 401 `errorCode 3`; no retry storm | +| B3 | High | Wrong domain | Login with `domain=velotrade` | Fails (401/3); retry with `default` succeeds — proves domain quirk | +| B4 | High | Login rate limit | 5 rapid logins | No hard 429; if 429, backoff honored (spec: 1 login/s default) | +| B5 | High | Token refresh on 401 | Call `request()` with expired token | One re-authenticate, one retry, success; exactly one extra login | +| B6 | High | **Auth header used for REST** | Inspect request headers on a real call | `Authorization: DXAPI ` present (see gap G1 — SDK sends `X-Auth-Token` only) | +| B7 | Med | User-Agent present | Inspect headers | Non-empty `User-Agent` on every request (spec requirement) | +| B8 | Med | `POST /ping` session validation | Send ping with valid token | Returns `200` and/or `sessionToken` (can refresh token) | +| B9 | Med | Logout | `SessionHandler.logout()` | `POST /logout`; local token cleared; subsequent calls re-auth | + +### Phase C — REST Read-Only API (credentials) + +| ID | Pri | Test | Steps | Expected result | +|---|---|---|---|---| +| C1 | High | Account discovery | `GET /users` via `request("GET","/users")` | JSON with account record containing full code `default:`; matches `DXTRADE_ACCOUNT` | +| C2 | High | Metrics | `GET /accounts/{enc}/metrics` | `account`, `equity`, `balance`, `margin`, `openPL`, counts; keys match `AccountMetrics` | +| C3 | High | Positions | `GET /accounts/{enc}/positions` | Empty list (fresh account) with 200 | +| C4 | High | Orders | `GET /accounts/{enc}/orders` and `/orders/history` | 200 (possibly empty); history contains any prior fills | +| C5 | High | Instrument discovery | `GET /accounts/{enc}/instruments/query?symbols=BTC` and no-arg variant | Symbols with `tradingStatus`, min/max order size, `marginRate`, `assetClass`; multi-asset across crypto/forex/equities/indices | +| C6 | Med | Instrument quantity units | Compare `minOrderSize` for EURUSD | Confirm lots vs base units; record conversion (blog: lots ×100,000 for forex) | +| C7 | Med | Market data snapshot | `POST /marketdata` `{symbols:[…], eventTypes:[{type:"Quote",format:"COMPACT"}]}` | Quote events for subscribed symbols | +| C8 | Med | Bad path shape | `GET /accounts/{code}/orders/` (trailing slash) or unencoded colon | Correctly reports 404/400; test verifies URL-encoding of `:` as `%3A` | +| C9 | Low | Rate limit surface | Burst 20 instrument queries | No 429; if 429, `Retry-After` honored, no blind retry | + +### Phase D — Push API / WebSocket (credentials) + +| ID | Pri | Test | Steps | Expected result | +|---|---|---|---|---| +| D1 | High | Market-data socket connects | `subscribe("quotes", cb, md_url)` | Handshake OK with `additional_headers` strategy; connection healthy | +| D2 | High | Quote subscription streams | `send_market_data_subscription(["EUR/USD","BTCUSD"], account)` | Within N s, callback receives `{"type":"MarketData",…,"payload":{"events":[…bid/ask…]}}`; `inReplyTo` matches `requestId` | +| D3 | High | **Subscription without `timestamp` accepted?** | Send SDK-shaped message (no `timestamp` field) | Record result: if `Reject`/error 32 arrives, document required field (gap G4) and add test for corrected message | +| D4 | High | Ping/pong auto-extension | Stay connected; observe `PingRequest` | SDK replies `{"type":"Ping","session","timestamp"}`; `get_ping_stats()["quotes"]["ping_responses_sent"]` increments; `get_session_health()` ≥ 1.0 | +| D5 | High | Portfolio socket connects | `subscribe("portfolio", cb, biz_url)` | Connects at `wss://…/dxsca-web/?format=JSON` | +| D6 | High | Portfolio subscription streams | `send_portfolio_subscription(account)` with SDK payload | **Expected mismatch** — server Rejects (error 32) because payload shape differs (gap G5). Fix payload to `{requestType:"LIST",accounts:[…]}` (+timestamp), then assert `{"type":"AccountPortfolios","payload":{"portfolios":[…]}}` snapshot arrives | +| D7 | High | Portfolio updates on order | Place small order (Phase E), watch portfolio | Full portfolio snapshot (no diffs) with new position/working order; `version` increments | +| D8 | Med | Explicit close subscription | Send `AccountPortfoliosCloseSubscriptionRequest` with `refRequestId` | `AccountPortfoliosSubscriptionClosed`; no further portfolio messages | +| D9 | Med | Multiple sessions one channel | Subscribe with two account codes on one socket | Independent streams, no message interleaving | +| D10 | Med | `RequestType=ALL` rate limit | Issue `RequestType=ALL` subscriptions >1/min | `Reject` 429; verify backoff behavior | +| D11 | Low | Compression param | Connect with `compression=gzip` | Either works transparently or server ignores; record actual behavior | +| D12 | Low | Backpressure (1013) | Slow consumer under heavy quotes | Connection closes 1013; transport surfaces error; no auto-flood of reconnects | + +### Phase E — Order Lifecycle (demo/sandbox ONLY) + +| ID | Pri | Test | Steps | Expected result | +|---|---|---|---|---| +| E1 | High | Market order open | `POST /accounts/{enc}/orders` with `orderCode`, `type=MARKET`, `side=BUY`, `instrument`, `quantity`, `tif=GTC` | 200 ack with order id; then order reaches `COMPLETED` and a position appears (confirm via C3 + Push) | +| E2 | High | Unique orderCode enforced | Repeat same `orderCode` | `409 errorCode 100`; original order untouched | +| E3 | High | Duplicate idempotency | Retry E1 with new `orderCode` after timeout | At most one fill (no double execution) — reconcile via history | +| E4 | High | Close position | `positionEffect=CLOSE`, `positionCode`, opposite `side`, same instrument, new `orderCode`, `tif=GTC`, omit `quantity` for full close | Position gone; no residual working/protective orders; verify via C3/C4 | +| E5 | High | Malformed order | `type=MARKET` with `limitPrice` set, or missing `quantity` | `400 errorCode 33`; no state change | +| E6 | High | Invalid instrument/symbol | Order on unlisted symbol | `400 errorCode 32`; clear error | +| E7 | Med | LIMIT order | `type=LIMIT` with realistic `limitPrice` | `WORKING` state; cancelable | +| E8 | Med | Cancel working order | `DELETE/PUT /accounts/{enc}/orders/{order}` per spec | Order `CANCELED`; position never opens | +| E9 | Med | STOP order + TP/SL | STOP order with `stopPrice`; attach TP/SL | Triggers per rules; protective orders visible in portfolio | +| E10 | Low | Bracket/group order | `POST /accounts/{enc}/orders/group` | Accepted or documented unsupported; record | +| E11 | High | Rules compliance | Compare API state after E1–E9 against Velotrade eval rules (daily loss, max drawdown) | No rule breach; note that eval rules bind API trades identically to manual | +| E12 | High | Flat-state smoke | Isolated connectivity smoke: open+close, assert flat | Total positions and working orders == 0 at end | + +### Phase F — Resilience and Shutdown + +| ID | Pri | Test | Steps | Expected result | +|---|---|---|---|---| +| F1 | High | Clean shutdown sequence | Stop new actions → reconcile → confirm flat → explicit close subscriptions → close sockets → REST logout | No orphan orders; `logout` returns; no exception on teardown | +| F2 | Med | WS drop + reconnect | Kill socket mid-subscription | Reconnect with backoff; **fresh REST snapshot before trusting local state** (blog guidance) | +| F3 | Med | Session expiry mid-run | Force token expiry (shorten `_token_expires_at`) | Next request re-authenticates; Push subscriptions resubscribed with new session | +| F4 | Med | Process interruption | SIGINT during connected stream | Socket/task cleanup; no zombie tasks; no partial subscriptions left | +| F5 | Low | Backoff + jitter | 3 failed connects | Delays grow (base_delay, exponential, jitter) — assert monotonic-ish | + +### Phase G — Security and Hygiene + +| ID | Pri | Test | Steps | Expected result | +|---|---|---|---|---| +| G-S1 | High | No secret leakage | Run A–F with `LOG_LEVEL=DEBUG` | No username/password/`sessionToken`/`Authorization` value in logs | +| G-S2 | High | Credentials `repr=False` | `repr(SessionCredentials(...))` | No password/token visible | +| G-S3 | High | `.env` gitignored | `git check-ignore .env` | Ignored; only `.env.example` committed | +| G-S4 | Med | `Authorization` header not logged | HTTP-level debug capture | Header values redacted | +| G-S5 | Med | No blind trade retries | Force 429/409 on order | Client surfaces error; does not auto-resubmit (409/100, 429 semantics) | + +--- + +## 6. Known SDK Gaps vs Velotrade (adaptation work — fixes needed before Phase C/D pass) + +| # | Gap | Evidence | Impact | Suggested fix | +|---|---|---|---|---| +| G1 | `transport.request()` sends only `X-Auth-Token`; Velotrade requires `Authorization: DXAPI ` | OpenAPI security scheme; Velotrade docs; `auth.py:228-229` sends both, `transport.py:195` sends one | All REST reads fail (401) | Add `Authorization: DXAPI` header in `transport.request()`; keep `X-Auth-Token` for legacy brokers (send both, like `SessionHandler`) | +| G2 | `DXTRADE_WS_MARKET_DATA_URL` / `DXTRADE_WS_PORTFOLIO_URL` documented but never read by `env_config.py` | `config.py:106-107` fields default None; no env read in `env_config.py`; `.env.example:17-20` documents them | `subscribe()` raises "No WebSocket URL configured" from env-only setup | Read the two vars in `_load_websocket_from_env()`; populate `market_data_url`/`portfolio_url` | +| G3 | `WebSocketConfig.get_market_data_url()/get_portfolio_url()` hardcode `{ws_base}/ws{path}` | `config.py:136,155` | Wrong URLs for Velotrade (`/md`, and `/` with trailing slash); never `/ws…` | Don't inject `/ws`; honor explicit URLs/paths verbatim | +| G4 | Push envelope `timestamp` omitted by SDK subscription builders | Push spec marks `timestamp` **required**; SDK `send_market_data_subscription`/`send_portfolio_subscription` omit it | Potential Reject (error 32) — must verify live (D3) | Add `timestamp` (ISO-8601 ms UTC) to all outbound Push messages | +| G5 | Portfolio subscription payload shape differs | Spec: `payload{requestType:"LIST", accounts:[…]}`; SDK: `payload{account, eventTypes:[{type:"Position",…}]}` | Portfolio subscription rejected; no portfolio stream | Build `{requestType, accounts}` payload; drop `eventTypes` for portfolio | +| G6 | Convenience methods hit dead paths | `transport.py:893-927` use `/accounts`, `/orders`, `/positions`, `/quotes`, `/time` → 404 | Those methods unusable on Velotrade | Point at `/users`, `/accounts/{code}/orders`, `/accounts/{code}/positions`, `/accounts/{code}/metrics`, `POST /marketdata`, `POST /ping` | +| G7 | No explicit Push close-subscription requests on shutdown | Spec requires `*CloseSubscriptionRequest` with `refRequestId`; SDK only closes the socket | Server keeps pushing / stale subs on reconnect | Send matching close requests in `unsubscribe()`/shutdown | +| G8 | Transport lacks reconnect with exponential backoff + jitter | `transport.py` `_websocket_handler` exits on failure; blog mandates backoff+jitter | Stream dies permanently on transient drop | Add bounded reconnect loop (reuse `RetryConfig`) | +| G9 | SDK default endpoints `/time`; Velotrade has no `/time` | OpenAPI: only `/ping` | `get_server_time()` 404 | Map to `POST /ping` | + +--- + +## 7. Exit Criteria + +1. A1–A7, B1, B2, B6, B7, B8, C1–C8, D1–D4, D6(fixed), D7, D8, E1–E6, E12, F1, G-S1–G-S3 all pass on the credentialed account. +2. Gaps G1–G9 confirmed resolved or explicitly waived with a tracked issue + workaround documented. +3. A run of the full suite leaves the account **flat** (zero positions, zero working orders) and logs out. +4. `pytest` (offline suite) stays green; live suite is opt-in via marker/flag. +5. No credentials in logs, artifacts, or committed files. + +--- + +## 8. Tooling and CI Notes + +- Offline tests must not touch the network; mock at the `aiohttp`/`websockets` boundary (conftest already mocks `httpx` — extend pattern to `aiohttp`). +- Record live exchanges once as VCR-style fixtures (request/response JSON, headers redacted) to keep CI hermetic and enable regression on message shapes (D2/D6). +- Add `pytest -m live` exclusion in CI; keep contract probes (A5–A7) in the default run. +- Timeouts: use generous WS waits (≥ server ping interval) — ping interval configurable via `DXTRADE_WS_PING_INTERVAL` (default 45 s); tests that need a fast ping should lower it. + +--- + +## 9. References + +- Velotrade API article: https://velotrade.com/blog/dxtrade-api-algo-trading +- Velotrade API access page: https://velotrade.com/api-access +- Velotrade developer portal (live): https://dx.velotrade.com/developers/ (REST API, Push API specs) +- Velotrade Swagger (live): https://dx.velotrade.com/dxsca-web/api/swagger.html — spec at `/dxsca-web/swagger/openapi.json` +- Velotrade DXtrade AI knowledge base: https://velotrade.com/downloads/velotrade-dxtrade-ai-kb.zip +- Trading terminal: https://dx.velotrade.com/ (login) | Portal: https://portal.velotrade.com/ +- Repo AGENTS.md "Known Issues" for the broken high-level layer (out of scope here) + +> Discovery date: 2026-08-14. Velotrade can change endpoints/behaviour — re-run Phase A probes and re-read the developer portal before a release; rules pages are controlling for trading rules. diff --git a/examples/stream_quotes.py b/examples/stream_quotes.py new file mode 100644 index 0000000..ba0e4dc --- /dev/null +++ b/examples/stream_quotes.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +Example: Stream real-time quotes using the high-level utils. + +Usage: + PYTHONPATH=src venv/Scripts/python.exe examples/stream_quotes.py + PYTHONPATH=src venv/Scripts/python.exe examples/stream_quotes.py \ + --symbols BTCUSDT ETHUSDT --duration 15 +""" + +import argparse +import asyncio +import sys + +# Windows consoles default to cp1252, which cannot encode the emoji used in +# the output; reconfigure stdout so printing never raises. +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + +from dxtrade import create_transport +from dxtrade.utils import stream_quotes + + +async def main() -> None: + parser = argparse.ArgumentParser(description="Stream DXTrade quotes.") + parser.add_argument( + "--symbols", + nargs="+", + default=None, + help="Symbols to stream (default: auto-discovered)", + ) + parser.add_argument( + "--duration", + type=float, + default=30.0, + help="Stream duration in seconds (default: 30)", + ) + args = parser.parse_args() + + transport = create_transport() + try: + + def on_quote(event): + print( + f"📈 {event.get('symbol')}: bid={event.get('bid')} ask={event.get('ask')}" + ) + + events = await stream_quotes( + transport, + symbols=args.symbols, + duration=args.duration, + on_quote=on_quote, + ) + print(f"✅ Streamed {len(events)} quote events in {args.duration}s") + finally: + await transport.close() + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n⏹️ Stopped by user") + sys.exit(0) diff --git a/examples/trade_smoke.py b/examples/trade_smoke.py new file mode 100644 index 0000000..88c5bcf --- /dev/null +++ b/examples/trade_smoke.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +Example: Open and close a small position using the high-level utils. + +Usage: + # Dry run (discovery + sizing only, no order) + PYTHONPATH=src venv/Scripts/python.exe examples/trade_smoke.py --dry-run + + # Open 0.001 BTC buy, hold 30s, close, verify flat + PYTHONPATH=src venv/Scripts/python.exe examples/trade_smoke.py --symbol BTCUSDT + + # Open with a $10 stop loss, hold 60s, close + PYTHONPATH=src venv/Scripts/python.exe examples/trade_smoke.py \ + --symbol BTCUSDT --stop-loss 10 --hold 60 + +WARNING: this trades real money on the account configured in .env. +""" + +import argparse +import asyncio +import json +import sys + +# Windows consoles default to cp1252, which cannot encode the emoji used in +# the output; reconfigure stdout so printing never raises. +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + +from dxtrade import create_transport +from dxtrade.utils import account_is_flat +from dxtrade.utils import close_position +from dxtrade.utils import flatten +from dxtrade.utils import open_position +from dxtrade.utils import resolve_account + + +async def main() -> None: + parser = argparse.ArgumentParser(description="Open/close a small DXTrade position.") + parser.add_argument( + "--symbol", default="BTCUSDT", help="Symbol to trade (default: BTCUSDT)" + ) + parser.add_argument("--side", default="BUY", choices=["BUY", "SELL"]) + parser.add_argument( + "--quantity", + type=float, + default=None, + help="Quantity (default: instrument minimum)", + ) + parser.add_argument( + "--stop-loss", + type=float, + default=None, + help="Max loss in account currency (optional)", + ) + parser.add_argument( + "--hold", + type=float, + default=30.0, + help="Seconds to hold before closing (default: 30)", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Only discover and size; place no orders" + ) + args = parser.parse_args() + + transport = create_transport() + try: + account = await resolve_account(transport) + print(f"🔑 Account: {account}") + + if args.dry_run: + print("🧪 DRY RUN — no orders will be placed") + else: + opened = await open_position( + transport, + symbol=args.symbol, + side=args.side, + quantity=args.quantity, + stop_loss=args.stop_loss, + account=account, + ) + position = opened["position"] + print( + f"✅ Opened {position.get('side')} {position.get('symbol')} " + f"qty={position.get('quantity')} @ {position.get('openPrice')} " + f"(code {position.get('positionCode')})" + ) + if opened["stop_order"]: + print(f"🛑 Stop loss order placed: {json.dumps(opened['stop_order'])}") + elif args.stop_loss: + print("⚠️ Stop loss was requested but not confirmed") + + print(f"⏱️ Holding {args.hold}s...") + await asyncio.sleep(args.hold) + + closed = await close_position( + transport, account=account, position_code=position["positionCode"] + ) + print( + f"✅ Closed position {position['positionCode']}: {json.dumps(closed['order'])}" + ) + + await flatten(transport, account=account) + flat = await account_is_flat(transport, account=account) + print(f"✅ Account flat: {flat}") + if not flat: + print("❌ Account NOT flat — check positions/orders") + finally: + await transport.close() + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n⏹️ Stopped by user") + sys.exit(0) diff --git a/pyproject.toml b/pyproject.toml index 6fc057d..3e3fd60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "pydantic>=2.5.0", "python-dotenv>=1.0.0", "typing-extensions>=4.8.0", + "httpx>=0.24.0", ] [project.optional-dependencies] diff --git a/src/dxtrade/auth.py b/src/dxtrade/auth.py index 294053c..22d630a 100644 --- a/src/dxtrade/auth.py +++ b/src/dxtrade/auth.py @@ -27,7 +27,7 @@ class AuthHandler(ABC): """Base class for authentication handlers.""" - + def __init__(self, credentials: AnyCredentials) -> None: """Initialize auth handler. @@ -63,10 +63,32 @@ def get_auth_type(self) -> AuthType: Authentication type """ + def get_auth_headers( + self, + method: str = "", + path: str = "", + body: str = "", + ) -> Dict[str, str]: + """Return authentication headers for an outgoing request. + + Transport layers use this to attach the handler's authentication + scheme without depending on a specific HTTP client. Subclasses + override this to provide their own header conventions. + + Args: + method: HTTP method (used by signature-based schemes) + path: Request path including query (used by signature-based schemes) + body: Raw request body (used by signature-based schemes) + + Returns: + Headers to add to the request + """ + return {} + class BearerTokenHandler(AuthHandler): """Bearer token authentication handler.""" - + def __init__(self, credentials: BearerTokenCredentials) -> None: """Initialize bearer token handler. @@ -97,6 +119,24 @@ async def authenticate( request.headers["Authorization"] = f"Bearer {self.credentials.token}" return request + def get_auth_headers( + self, + method: str = "", + path: str = "", + body: str = "", + ) -> Dict[str, str]: + """Return authentication headers for an outgoing request. + + Args: + method: HTTP method (unused for bearer token) + path: Request path (unused for bearer token) + body: Raw request body (unused for bearer token) + + Returns: + Headers to add to the request + """ + return {"Authorization": f"Bearer {self.credentials.token}"} + def get_auth_type(self) -> AuthType: """Get the authentication type. @@ -108,7 +148,7 @@ def get_auth_type(self) -> AuthType: class HMACHandler(AuthHandler): """HMAC authentication handler.""" - + def __init__(self, credentials: HMACCredentials) -> None: """Initialize HMAC handler. @@ -137,22 +177,43 @@ async def authenticate( Authenticated request """ timestamp = str(int(time.time() * 1000)) - + # Prepare signature components method = request.method.upper() path = str(request.url.path) if request.url.query: path += f"?{request.url.query}" - + body = "" if request.content: body = request.content.decode("utf-8") - + + request.headers.update(self.get_auth_headers(method=method, path=path, body=body)) + return request + + def get_auth_headers( + self, + method: str = "", + path: str = "", + body: str = "", + ) -> Dict[str, str]: + """Return authentication headers for an outgoing request. + + Args: + method: HTTP method + path: Request path including query + body: Raw request body + + Returns: + Headers to add to the request + """ + timestamp = str(int(time.time() * 1000)) + # Create signature string - signature_string = f"{timestamp}{method}{path}{body}" + signature_string = f"{timestamp}{method.upper()}{path}{body}" if self.credentials.passphrase: signature_string += self.credentials.passphrase - + # Generate HMAC signature signature = hmac.new( self.credentials.secret_key.encode("utf-8"), @@ -160,16 +221,18 @@ async def authenticate( hashlib.sha256, ).digest() signature_b64 = b64encode(signature).decode("utf-8") - + # Add authentication headers - request.headers["DX-API-KEY"] = self.credentials.api_key - request.headers["DX-API-TIMESTAMP"] = timestamp - request.headers["DX-API-SIGNATURE"] = signature_b64 - + headers = { + "DX-API-KEY": self.credentials.api_key, + "DX-API-TIMESTAMP": timestamp, + "DX-API-SIGNATURE": signature_b64, + } + if self.credentials.passphrase: - request.headers["DX-API-PASSPHRASE"] = self.credentials.passphrase - - return request + headers["DX-API-PASSPHRASE"] = self.credentials.passphrase + + return headers def get_auth_type(self) -> AuthType: """Get the authentication type. @@ -182,7 +245,7 @@ def get_auth_type(self) -> AuthType: class SessionHandler(AuthHandler): """Session-based authentication handler.""" - + def __init__(self, credentials: SessionCredentials) -> None: """Initialize session handler. @@ -220,13 +283,12 @@ async def authenticate( # Check if we need to get/refresh session token if not self._session_token or self._is_token_expired(): await self._refresh_session_token(client) - + if not self._session_token: raise DXtradeAuthenticationError("Failed to obtain session token") - - # Add both X-Auth-Token and Authorization headers as shown in the example - request.headers["X-Auth-Token"] = self._session_token - request.headers["Authorization"] = f"DXAPI {self._session_token}" + + # Both X-Auth-Token and Authorization headers are required by DXTrade + request.headers.update(self.get_auth_headers()) return request async def _refresh_session_token(self, client: httpx.AsyncClient) -> None: @@ -244,28 +306,28 @@ async def _refresh_session_token(self, client: httpx.AsyncClient) -> None: "password": self.credentials.password, "domain": self.credentials.domain or "default", } - + # Use /login endpoint as per the integration guide response = await client.post("/login", json=login_data) response.raise_for_status() - + data = response.json() - + # Get sessionToken from response (as per integration guide) self._session_token = data.get("sessionToken") - + if not self._session_token: error_msg = data.get("message") or "Login failed - no session token received" raise DXtradeAuthenticationError(error_msg) - + # Store expiration time if provided, otherwise default to 1 hour expires_in = data.get("expiresIn", 3600) self._token_expires_at = time.time() + expires_in - 300 # With 5 min buffer self._last_login = time.time() - + # Store accounts for reference self.accounts = data.get("accounts", []) - + except httpx.HTTPError as e: raise DXtradeAuthenticationError(f"Login request failed: {e}") from e @@ -277,11 +339,11 @@ def _is_token_expired(self) -> bool: """ if not self._token_expires_at or not self._last_login: return True - + # Re-login if session is older than 1 hour (as per example) if (time.time() - self._last_login) > 3600: return True - + return time.time() >= self._token_expires_at def get_auth_type(self) -> AuthType: @@ -291,7 +353,33 @@ def get_auth_type(self) -> AuthType: Session auth type """ return AuthType.SESSION - + + def get_auth_headers( + self, + method: str = "", + path: str = "", + body: str = "", + ) -> Dict[str, str]: + """Return authentication headers for an outgoing request. + + The DXTrade API requires the session token in both the + ``X-Auth-Token`` and ``Authorization: DXAPI `` headers. + + Args: + method: HTTP method (unused for session auth) + path: Request path (unused for session auth) + body: Raw request body (unused for session auth) + + Returns: + Headers to add to the request + """ + if not self._session_token: + return {} + return { + "X-Auth-Token": self._session_token, + "Authorization": f"DXAPI {self._session_token}", + } + def get_session_token(self) -> Optional[str]: """Get the current session token. @@ -308,7 +396,7 @@ async def logout(self, client: httpx.AsyncClient) -> None: """ if not self._session_token: return - + try: # Try to invalidate token on server (as per integration guide) headers = { @@ -328,7 +416,7 @@ async def logout(self, client: httpx.AsyncClient) -> None: class AuthFactory: """Factory for creating authentication handlers.""" - + _handlers: Dict[AuthType, type[AuthHandler]] = { AuthType.BEARER_TOKEN: BearerTokenHandler, AuthType.HMAC: HMACHandler, @@ -356,7 +444,7 @@ def create_handler( handler_class = self._handlers.get(auth_type) if not handler_class: raise DXtradeConfigurationError(f"Unsupported auth type: {auth_type}") - + return handler_class(credentials) @classmethod @@ -380,4 +468,4 @@ def get_supported_types(cls) -> list[AuthType]: Returns: List of supported auth types """ - return list(cls._handlers.keys()) \ No newline at end of file + return list(cls._handlers.keys()) diff --git a/src/dxtrade/config.py b/src/dxtrade/config.py index b86193a..4385a9f 100644 --- a/src/dxtrade/config.py +++ b/src/dxtrade/config.py @@ -33,7 +33,7 @@ class Features: auto_reconnect: bool = True rate_limiting: bool = True automatic_retry: bool = True - + def to_dict(self) -> Dict[str, bool]: """Convert features to dictionary.""" return { @@ -52,49 +52,49 @@ class Endpoints: login: str = '/login' logout: str = '/logout' refresh_token: str = '/refresh' - + # Market data endpoints market_data: str = '/marketdata' quotes: str = '/quotes' candles: str = '/candles' instruments: str = '/instruments' - + # Account endpoints account: str = '/account' accounts: str = '/accounts' portfolio: str = '/portfolio' balance: str = '/balance' metrics: str = '/accounts/metrics' - + # Trading endpoints orders: str = '/orders' orders_history: str = '/accounts/orders/history' positions: str = '/accounts/positions' trades: str = '/trades' history: str = '/history' - + # System endpoints time: str = '/time' status: str = '/status' version: str = '/version' conversion_rates: str = '/conversionRates' - + # WebSocket endpoints (legacy paths) ws_market_data: str = '/md' ws_portfolio: str = '/' - + def get_endpoint(self, name: str, base_url: Optional[str] = None) -> str: """Get endpoint by name with fallback.""" endpoint = getattr(self, name, f'/{name}') - + # If it's already a complete URL, return as-is if endpoint.startswith(('http://', 'https://')): return endpoint - + # If we have a base URL, construct the full URL if base_url: return f"{base_url.rstrip('/')}{endpoint if endpoint.startswith('/') else '/' + endpoint}" - + # Return the path/endpoint as-is return endpoint @@ -105,7 +105,7 @@ class WebSocketConfig: # Explicit URLs (preferred) market_data_url: Optional[str] = None portfolio_url: Optional[str] = None - + # Legacy configuration (fallback) base_url: Optional[str] = None market_data_path: str = '/md' @@ -115,17 +115,17 @@ class WebSocketConfig: reconnect_attempts: int = 5 reconnect_delay: float = 1.0 # seconds max_message_size: int = 1024 * 1024 # 1MB - + def get_market_data_url(self, base_url: Optional[str] = None) -> str: """Get complete market data WebSocket URL.""" # Use explicit URL if available if self.market_data_url: return self.market_data_url - + # Fallback to constructing URL from base + path if not base_url and not self.base_url: raise ValueError("No market data WebSocket URL available") - + ws_base = self.base_url or base_url.replace('https://', 'wss://').replace('http://', 'ws://') # Ensure path has format parameter path = self.market_data_path @@ -133,18 +133,18 @@ def get_market_data_url(self, base_url: Optional[str] = None) -> str: path = f"{path}?format={self.format}" elif 'format=' not in path: path = f"{path}&format={self.format}" - return f"{ws_base}/ws{path}" - + return f"{ws_base}{path}" + def get_portfolio_url(self, base_url: Optional[str] = None) -> str: """Get complete portfolio WebSocket URL.""" # Use explicit URL if available if self.portfolio_url: return self.portfolio_url - + # Fallback to constructing URL from base + path if not base_url and not self.base_url: raise ValueError("No portfolio WebSocket URL available") - + ws_base = self.base_url or base_url.replace('https://', 'wss://').replace('http://', 'ws://') # Ensure path has format parameter path = self.portfolio_path @@ -152,7 +152,7 @@ def get_portfolio_url(self, base_url: Optional[str] = None) -> str: path = f"{path}?format={self.format}" elif 'format=' not in path: path = f"{path}&format={self.format}" - return f"{ws_base}/ws{path}" + return f"{ws_base}{path}" @dataclass @@ -186,25 +186,25 @@ class RetryConfig: class AuthConfig: """Authentication configuration.""" type: AuthType - + # Credentials auth username: Optional[str] = None password: Optional[str] = None domain: str = 'default' - + # Session auth session_token: Optional[str] = None auto_refresh: bool = True refresh_before_expiry: int = 300 # seconds - + # Bearer auth bearer_token: Optional[str] = None - + # HMAC auth api_key: Optional[str] = None api_secret: Optional[str] = None passphrase: Optional[str] = None - + def validate(self) -> None: """Validate authentication configuration.""" if self.type == AuthType.CREDENTIALS: @@ -229,49 +229,49 @@ class SDKConfig: base_url: Optional[str] = None timeout: int = 30000 # milliseconds user_agent: str = 'dxtrade-python-sdk/2.0.0' - + # Authentication auth: AuthConfig = field(default_factory=lambda: AuthConfig(type=AuthType.CREDENTIALS)) - + # Features features: Features = field(default_factory=Features) - + # Endpoints endpoints: Endpoints = field(default_factory=Endpoints) - + # WebSocket websocket: Optional[WebSocketConfig] = field(default_factory=WebSocketConfig) - + # Rate limiting rate_limit: RateLimitConfig = field(default_factory=RateLimitConfig) - + # Retry behavior retry: RetryConfig = field(default_factory=RetryConfig) - + # Logging log_level: str = 'INFO' log_requests: bool = False log_responses: bool = False - + # Account configuration account: Optional[str] = None - + def validate(self) -> None: """Validate the complete configuration.""" if not self.base_url: raise ValueError("base_url is required") - + if not self.base_url.startswith(('http://', 'https://')): raise ValueError("base_url must start with http:// or https://") - + self.auth.validate() - + if self.timeout <= 0: raise ValueError("timeout must be positive") - + if self.features.websocket and not self.websocket: self.websocket = WebSocketConfig() - + def to_dict(self) -> Dict[str, Any]: """Convert configuration to dictionary.""" return { @@ -288,21 +288,21 @@ def to_dict(self) -> Dict[str, Any]: 'features': self.features.to_dict(), 'log_level': self.log_level, } - + @classmethod def from_dict(cls, data: Dict[str, Any]) -> 'SDKConfig': """Create configuration from dictionary.""" config = cls() - + if 'environment' in data: config.environment = Environment(data['environment']) - + if 'base_url' in data: config.base_url = data['base_url'] - + if 'timeout' in data: config.timeout = data['timeout'] - + if 'auth' in data: auth_data = data['auth'] auth_type = AuthType(auth_data.get('type', 'credentials')) @@ -317,18 +317,18 @@ def from_dict(cls, data: Dict[str, Any]) -> 'SDKConfig': api_secret=auth_data.get('api_secret'), passphrase=auth_data.get('passphrase') ) - + if 'features' in data: config.features = Features(**data['features']) - + if 'endpoints' in data: config.endpoints = Endpoints(**data['endpoints']) - + if 'websocket' in data: config.websocket = WebSocketConfig(**data['websocket']) - + return config # Alias for backward compatibility -DXTradeConfig = SDKConfig \ No newline at end of file +DXTradeConfig = SDKConfig diff --git a/src/dxtrade/env_config.py b/src/dxtrade/env_config.py index 2411396..cd987ae 100644 --- a/src/dxtrade/env_config.py +++ b/src/dxtrade/env_config.py @@ -240,6 +240,13 @@ def _load_websocket_from_env() -> WebSocketConfig: if ws_url := os.getenv('DXTRADE_WS_URL'): ws_config.base_url = ws_url.rstrip('/') + # Explicit per-channel URLs take precedence over base + path + if market_data_url := os.getenv('DXTRADE_WS_MARKET_DATA_URL'): + ws_config.market_data_url = market_data_url + + if portfolio_url := os.getenv('DXTRADE_WS_PORTFOLIO_URL'): + ws_config.portfolio_url = portfolio_url + if market_data_path := os.getenv('DXTRADE_WS_MARKET_DATA_PATH'): ws_config.market_data_path = market_data_path diff --git a/src/dxtrade/transport.py b/src/dxtrade/transport.py index 9e76777..0e5cc90 100644 --- a/src/dxtrade/transport.py +++ b/src/dxtrade/transport.py @@ -16,7 +16,7 @@ import time from datetime import datetime from typing import Any, Callable, Dict, Optional, Union -from urllib.parse import urljoin +from urllib.parse import quote, urljoin import aiohttp import websockets @@ -30,9 +30,21 @@ logger = logging.getLogger(__name__) +def _utc_timestamp() -> str: + """Return the current UTC time in the DXTrade Push timestamp format. + + DXTrade Push messages carry a ``timestamp`` field in ISO-8601 UTC + format with milliseconds (e.g. ``2026-08-14T12:34:56.789Z``). + + Returns: + Current UTC time with milliseconds + """ + return datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ")[:-3] + "Z" + + class DXTradeTransport: """Minimal DXTrade transport client for raw API access.""" - + def __init__(self, config=None): """Initialize transport client. @@ -42,11 +54,11 @@ def __init__(self, config=None): if config is None: load_dotenv() config = load_config_from_env() - + self.config = config self.base_url = config.base_url self.websocket_url = getattr(config, 'websocket_url', None) - + # Session authentication self.credentials = SessionCredentials( username=config.auth.username, @@ -54,31 +66,31 @@ def __init__(self, config=None): domain=config.auth.domain ) self.auth_handler = SessionHandler(self.credentials) - + # HTTP session self._session: Optional[aiohttp.ClientSession] = None - + # WebSocket connections self._websockets: Dict[str, websockets.WebSocketClientProtocol] = {} self._subscriptions: Dict[str, Callable] = {} self._ws_tasks: Dict[str, asyncio.Task] = {} - + # Application-level ping/pong tracking self._ping_stats: Dict[str, Dict] = {} self._enable_ping_logging: bool = True - + # WebSocket connection strategy tracking self._successful_strategies: Dict[str, str] = {} - + # Log websockets library version for debugging self._log_websockets_version() - + def _log_websockets_version(self): """Log websockets library version for debugging compatibility issues.""" try: version = getattr(websockets, '__version__', 'unknown') logger.info(f"🔌 Using websockets library version: {version}") - + # Log compatibility information if version != 'unknown': major_version = int(version.split('.')[0]) if version.split('.')[0].isdigit() else 0 @@ -88,35 +100,35 @@ def _log_websockets_version(self): logger.debug("WebSocket library supports legacy extra_headers parameter") else: logger.warning("WebSocket library version may have compatibility issues - consider upgrading to 11.0+") - + except Exception as e: logger.debug(f"Could not determine websockets version: {e}") - + async def __aenter__(self): """Async context manager entry.""" await self._ensure_session() return self - + async def __aexit__(self, exc_type, exc_val, exc_tb): """Async context manager exit.""" await self.close() - + async def _ensure_session(self): """Ensure HTTP session exists.""" if self._session is None: self._session = aiohttp.ClientSession() - + async def close(self): """Close all connections.""" # Close WebSocket connections for channel in list(self._websockets.keys()): await self.unsubscribe(channel) - + # Close HTTP session if self._session: await self._session.close() self._session = None - + async def authenticate(self) -> str: """Authenticate and return session token. @@ -127,43 +139,43 @@ async def authenticate(self) -> str: Exception: Authentication failed """ await self._ensure_session() - + # Manual authentication since auth handler expects different client login_data = { "username": self.credentials.username, "password": self.credentials.password, "domain": self.credentials.domain or "default", } - + # Use explicit login URL if available login_url = getattr(self.config.endpoints, 'login', '/login') if not login_url.startswith('http'): login_url = urljoin(self.base_url + '/', login_url.lstrip('/')) - + logger.debug(f"Authenticating at {login_url}") - + async with self._session.post(login_url, json=login_data) as response: response.raise_for_status() data = await response.json() - + # Get sessionToken from response session_token = data.get("sessionToken") if not session_token: error_msg = data.get("message") or "Login failed - no session token received" raise Exception(error_msg) - + # Store token in auth handler self.auth_handler._session_token = session_token self.auth_handler._last_login = time.time() self.auth_handler._token_expires_at = time.time() + 3600 # 1 hour - + logger.info("Authentication successful") return session_token - + async def request( - self, - method: str, - endpoint: str, + self, + method: str, + endpoint: str, **kwargs ) -> Union[Dict[str, Any], list, str]: """Make raw HTTP request with authentication. @@ -177,23 +189,22 @@ async def request( Raw response data (JSON parsed if possible) """ await self._ensure_session() - + # Ensure we have a valid session token token = self.auth_handler.get_session_token() if not token: token = await self.authenticate() - + # Build full URL if endpoint.startswith('http'): url = endpoint else: url = urljoin(self.base_url + '/', endpoint.lstrip('/')) - - # Add auth headers + + # Add auth headers from the auth handler (broker-specific scheme) headers = kwargs.pop('headers', {}) - if token: - headers['X-Auth-Token'] = token - + headers.update(self.auth_handler.get_auth_headers()) + # Make request logger.debug(f"Making {method} request to {url}") async with self._session.request(method, url, headers=headers, **kwargs) as response: @@ -201,22 +212,21 @@ async def request( if response.status == 401: logger.info("Got 401, refreshing session token") token = await self.authenticate() - + # Update headers with new token - if token: - headers['X-Auth-Token'] = token - + headers.update(self.auth_handler.get_auth_headers()) + # Retry with new token async with self._session.request(method, url, headers=headers, **kwargs) as retry_response: return await self._parse_response(retry_response) - + return await self._parse_response(response) - + async def _parse_response(self, response: aiohttp.ClientResponse) -> Union[Dict, list, str]: """Parse response, returning raw data.""" # Raise for HTTP errors response.raise_for_status() - + # Try to parse as JSON first content_type = response.headers.get('content-type', '') if 'json' in content_type: @@ -224,10 +234,10 @@ async def _parse_response(self, response: aiohttp.ClientResponse) -> Union[Dict, return await response.json() except Exception: pass - + # Fall back to text return await response.text() - + async def subscribe(self, channel: str, callback: Callable[[dict], None], ws_url: Optional[str] = None): """Subscribe to WebSocket channel with raw message forwarding. @@ -239,31 +249,51 @@ async def subscribe(self, channel: str, callback: Callable[[dict], None], ws_url if channel in self._websockets: logger.warning(f"Already subscribed to channel: {channel}") return - + # Use provided URL or build from config if ws_url is None: if hasattr(self.config, 'websocket') and self.config.websocket: - if channel == "quotes" or channel == "market_data": - ws_url = getattr(self.config.websocket, 'market_data_url', None) - else: - ws_url = getattr(self.config.websocket, 'portfolio_url', None) - - # Fallback to base URL construction - if not ws_url and hasattr(self.config.websocket, 'base_url'): - ws_url = self.config.websocket.base_url - + try: + if channel == "quotes" or channel == "market_data": + ws_url = self.config.websocket.get_market_data_url(self.base_url) + else: + ws_url = self.config.websocket.get_portfolio_url(self.base_url) + except ValueError: + ws_url = None + if not ws_url: raise ValueError(f"No WebSocket URL configured for channel: {channel}") - + logger.info(f"Subscribing to {channel} at {ws_url}") - + # Store subscription self._subscriptions[channel] = callback - + # Start WebSocket connection task task = asyncio.create_task(self._websocket_handler(channel, ws_url)) self._ws_tasks[channel] = task - + + async def wait_for_channel(self, channel: str, timeout: float = 30.0) -> bool: + """Wait until the WebSocket connection for a channel is established. + + ``subscribe()`` starts the connection in the background; use this to + wait until it is ready (e.g. before sending a subscription message). + + Args: + channel: Channel name (e.g. "quotes", "portfolio") + timeout: Maximum wait in seconds + + Returns: + True if the channel connected, False on timeout + """ + loop = asyncio.get_event_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + if channel in self._websockets: + return True + await asyncio.sleep(0.1) + return False + async def _establish_websocket_connection(self, ws_url: str, token: Optional[str], channel: str): """Establish WebSocket connection with multiple compatibility approaches. @@ -289,9 +319,9 @@ async def _establish_websocket_connection(self, ws_url: str, token: Optional[str ("subprotocol_auth", self._connect_with_subprotocol_auth), ("post_connection_auth", self._connect_with_post_connection_auth), ] - + last_error = None - + for approach_name, connect_func in connection_approaches: try: logger.debug(f"Trying WebSocket connection approach: {approach_name} for {channel}") @@ -301,45 +331,39 @@ async def _establish_websocket_connection(self, ws_url: str, token: Optional[str # Track successful strategy for this channel self._successful_strategies[channel] = approach_name return websocket - + except Exception as e: last_error = e logger.debug(f"WebSocket approach {approach_name} failed for {channel}: {e}") continue - + # All approaches failed logger.error(f"❌ All WebSocket connection approaches failed for {channel}") if last_error: logger.error(f"Last error: {last_error}") - + return None - + async def _connect_with_additional_headers(self, ws_url: str, token: Optional[str]): """Connect using additional_headers parameter (websockets 11.0+).""" try: - headers = {} - if token: - headers['X-Auth-Token'] = token - + headers = self.auth_handler.get_auth_headers() return await websockets.connect(ws_url, additional_headers=headers) except TypeError as e: if 'additional_headers' in str(e): raise Exception("additional_headers parameter not supported by this websockets version") raise - + async def _connect_with_extra_headers(self, ws_url: str, token: Optional[str]): - """Connect using extra_headers parameter (websockets 9.0-10.x).""" + """Connect using extra_headers parameter (websockets 9.0-10.x).""" try: - headers = {} - if token: - headers['X-Auth-Token'] = token - + headers = self.auth_handler.get_auth_headers() return await websockets.connect(ws_url, extra_headers=headers) except TypeError as e: if 'extra_headers' in str(e): raise Exception("extra_headers parameter not supported by this websockets version") raise - + async def _connect_with_subprotocol_auth(self, ws_url: str, token: Optional[str]): """Connect using subprotocol for authentication (fallback approach).""" try: @@ -347,46 +371,46 @@ async def _connect_with_subprotocol_auth(self, ws_url: str, token: Optional[str] if token: # Encode token in subprotocol (some servers support this) subprotocols = [f"auth.{token}"] - + return await websockets.connect(ws_url, subprotocols=subprotocols) except Exception as e: # Add context to subprotocol failures raise Exception(f"Subprotocol authentication failed: {e}") - + async def _connect_with_post_connection_auth(self, ws_url: str, token: Optional[str]): """Connect without headers and authenticate after connection (last resort).""" try: websocket = await websockets.connect(ws_url) - + if token: # Send authentication message after connection auth_message = { - "type": "authenticate", + "type": "authenticate", "token": token, "channel": "auth" } await websocket.send(json.dumps(auth_message)) - + # Wait for auth response with timeout try: response = await asyncio.wait_for(websocket.recv(), timeout=10.0) auth_response = json.loads(response) if isinstance(response, str) else response - + if isinstance(auth_response, dict) and auth_response.get('type') == 'auth_success': logger.debug("Post-connection authentication successful") else: logger.warning(f"Unexpected auth response: {auth_response}") - + except asyncio.TimeoutError: logger.warning("No authentication response received (continuing anyway)") except Exception as e: logger.warning(f"Post-connection auth error: {e} (continuing anyway)") - + return websocket - + except Exception as e: raise Exception(f"Post-connection authentication approach failed: {e}") - + async def _websocket_handler(self, channel: str, ws_url: str): """Handle WebSocket connection and messages.""" try: @@ -394,17 +418,17 @@ async def _websocket_handler(self, channel: str, ws_url: str): token = self.auth_handler.get_session_token() if not token: token = await self.authenticate() - + # Connect to WebSocket with compatibility fallbacks websocket = await self._establish_websocket_connection(ws_url, token, channel) if not websocket: raise Exception(f"Failed to establish WebSocket connection for {channel}") - + # Use connection in context manager style async with websocket: self._websockets[channel] = websocket logger.info(f"Connected to WebSocket for channel: {channel}") - + # Initialize ping stats for this channel self._ping_stats[channel] = { 'ping_requests_received': 0, @@ -413,10 +437,10 @@ async def _websocket_handler(self, channel: str, ws_url: str): 'last_ping_response': None, 'session_extensions': 0 } - + # Don't auto-send subscription - let user control it # await self._send_dxtrade_subscription(websocket, channel, token) - + # Listen for messages async for message in websocket: try: @@ -428,11 +452,11 @@ async def _websocket_handler(self, channel: str, ws_url: str): data = message else: data = message - + # Handle application-level ping/pong for session management if await self._handle_ping_pong(channel, data, websocket, token): continue # Skip forwarding ping/pong messages to user callback - + # Forward raw message to callback callback = self._subscriptions.get(channel) if callback: @@ -440,10 +464,10 @@ async def _websocket_handler(self, channel: str, ws_url: str): callback(data) except Exception as e: logger.error(f"Error in callback for {channel}: {e}") - + except Exception as e: logger.error(f"Error processing message for {channel}: {e}") - + except Exception as e: logger.error(f"WebSocket error for {channel}: {e}") finally: @@ -452,7 +476,7 @@ async def _websocket_handler(self, channel: str, ws_url: str): self._subscriptions.pop(channel, None) self._ping_stats.pop(channel, None) self._successful_strategies.pop(channel, None) - + async def _handle_ping_pong(self, channel: str, data: Union[dict, str], websocket: websockets.WebSocketClientProtocol, token: str) -> bool: """Handle application-level ping/pong for DXTrade session management. @@ -469,18 +493,18 @@ async def _handle_ping_pong(self, channel: str, data: Union[dict, str], websocke # Check if this is a PingRequest from server if isinstance(data, dict) and data.get("type") == "PingRequest": timestamp = datetime.now() - + # Update stats - stats = self._ping_stats.get(channel, {}) + stats = self._ping_stats.setdefault(channel, {}) stats['ping_requests_received'] = stats.get('ping_requests_received', 0) + 1 stats['last_ping_request'] = timestamp stats['session_extensions'] = stats.get('session_extensions', 0) + 1 - + # Log ping request activity if self._enable_ping_logging: logger.info(f"🔄 Received PingRequest on channel '{channel}' - extending session") logger.debug(f" Ping stats: {stats['ping_requests_received']} requests, {stats['session_extensions']} extensions") - + # Send DXTrade Ping response with session and timestamp ping_response = { "type": "Ping", @@ -488,50 +512,50 @@ async def _handle_ping_pong(self, channel: str, data: Union[dict, str], websocke "timestamp": timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ")[:-3] + "Z" # ISO format with milliseconds } await websocket.send(json.dumps(ping_response)) - + # Update response stats stats['ping_responses_sent'] = stats.get('ping_responses_sent', 0) + 1 stats['last_ping_response'] = timestamp - + # Log successful ping response if self._enable_ping_logging: logger.info(f"✅ Sent Ping response on channel '{channel}' - session extended") - + return True # Message was handled, don't forward to user callback - + # Check if this is a string-based ping request (alternative format) elif isinstance(data, str) and data.lower() in ["pingrequest", "ping_request"]: timestamp = datetime.now() - + # Update stats - stats = self._ping_stats.get(channel, {}) + stats = self._ping_stats.setdefault(channel, {}) stats['ping_requests_received'] = stats.get('ping_requests_received', 0) + 1 stats['last_ping_request'] = timestamp stats['session_extensions'] = stats.get('session_extensions', 0) + 1 - + # Log ping request activity if self._enable_ping_logging: logger.info(f"🔄 Received string PingRequest '{data}' on channel '{channel}' - extending session") - + # Send string-based Ping response await websocket.send("Ping") - + # Update response stats stats['ping_responses_sent'] = stats.get('ping_responses_sent', 0) + 1 stats['last_ping_response'] = timestamp - + # Log successful ping response if self._enable_ping_logging: logger.info(f"✅ Sent Ping response on channel '{channel}' - session extended") - + return True # Message was handled, don't forward to user callback - + return False # Not a ping/pong message - + except Exception as e: logger.error(f"Error handling ping/pong for channel '{channel}': {e}") return False - + async def unsubscribe(self, channel: str): """Unsubscribe from WebSocket channel.""" # Cancel task @@ -542,17 +566,17 @@ async def unsubscribe(self, channel: str): await task except asyncio.CancelledError: pass - + # Close WebSocket websocket = self._websockets.pop(channel, None) if websocket: await websocket.close() - + # Remove subscription self._subscriptions.pop(channel, None) - + logger.info(f"Unsubscribed from channel: {channel}") - + def enable_ping_logging(self, enabled: bool = True): """Enable or disable ping/pong activity logging. @@ -564,7 +588,7 @@ def enable_ping_logging(self, enabled: bool = True): logger.info("✅ DXTrade application-level ping/pong logging enabled") else: logger.info("❌ DXTrade application-level ping/pong logging disabled") - + def get_ping_stats(self, channel: Optional[str] = None) -> Union[Dict, Dict[str, Dict]]: """Get ping/pong statistics for session monitoring. @@ -577,7 +601,7 @@ def get_ping_stats(self, channel: Optional[str] = None) -> Union[Dict, Dict[str, if channel: return self._ping_stats.get(channel, {}) return self._ping_stats.copy() - + def get_session_health(self) -> Dict[str, Any]: """Get overall session health metrics for monitoring bridge status. @@ -587,10 +611,10 @@ def get_session_health(self) -> Dict[str, Any]: total_ping_requests = sum(stats.get('ping_requests_received', 0) for stats in self._ping_stats.values()) total_ping_responses = sum(stats.get('ping_responses_sent', 0) for stats in self._ping_stats.values()) total_extensions = sum(stats.get('session_extensions', 0) for stats in self._ping_stats.values()) - + active_channels = len(self._websockets) healthy_channels = len([ch for ch, ws in self._websockets.items() if self._is_websocket_healthy(ws)]) - + return { 'active_channels': active_channels, 'healthy_channels': healthy_channels, @@ -599,13 +623,13 @@ def get_session_health(self) -> Dict[str, Any]: 'total_ping_responses_sent': total_ping_responses, 'total_session_extensions': total_extensions, 'ping_response_success_rate': total_ping_responses / total_ping_requests if total_ping_requests > 0 else 1.0, - 'last_activity': max([stats.get('last_ping_response') for stats in self._ping_stats.values() + 'last_activity': max([stats.get('last_ping_response') for stats in self._ping_stats.values() if stats.get('last_ping_response')], default=None), 'channels': list(self._websockets.keys()), 'connection_strategies': self._successful_strategies.copy(), 'websockets_version': getattr(websockets, '__version__', 'unknown') } - + def _is_websocket_healthy(self, websocket: websockets.WebSocketClientProtocol) -> bool: """Check if WebSocket connection is healthy. @@ -640,7 +664,7 @@ def get_connection_strategies(self) -> Dict[str, str]: Dictionary mapping channel names to connection strategy names """ return self._successful_strategies.copy() - + def check_websockets_compatibility(self) -> Dict[str, Any]: """Check websockets library compatibility and provide recommendations. @@ -649,7 +673,7 @@ def check_websockets_compatibility(self) -> Dict[str, Any]: """ try: version = getattr(websockets, '__version__', 'unknown') - + if version == 'unknown': return { 'version': 'unknown', @@ -657,10 +681,10 @@ def check_websockets_compatibility(self) -> Dict[str, Any]: 'message': 'Cannot determine websockets library version', 'recommendations': ['Check websockets installation', 'Consider reinstalling websockets>=12.0'] } - + major_version = int(version.split('.')[0]) if version.split('.')[0].isdigit() else 0 minor_version = int(version.split('.')[1]) if len(version.split('.')) > 1 and version.split('.')[1].isdigit() else 0 - + if major_version >= 12: return { 'version': version, @@ -689,7 +713,7 @@ def check_websockets_compatibility(self) -> Dict[str, Any]: 'message': 'Poor compatibility - connection issues likely', 'recommendations': ['Upgrade to websockets>=12.0 immediately', 'Current version may cause connection failures'] } - + except Exception as e: return { 'version': 'error', @@ -697,7 +721,7 @@ def check_websockets_compatibility(self) -> Dict[str, Any]: 'message': f'Error checking websockets compatibility: {e}', 'recommendations': ['Check websockets installation', 'Reinstall websockets>=12.0'] } - + async def _send_dxtrade_subscription(self, websocket: websockets.WebSocketClientProtocol, channel: str, session_token: str): """Send DXTrade-specific subscription message. @@ -708,9 +732,9 @@ async def _send_dxtrade_subscription(self, websocket: websockets.WebSocketClient """ import uuid from datetime import datetime - + request_id = str(uuid.uuid4()) - + # Get account from config or environment account = getattr(self.config, 'account', None) if not account: @@ -718,13 +742,14 @@ async def _send_dxtrade_subscription(self, websocket: websockets.WebSocketClient domain = getattr(self.config.auth, 'domain', 'default') account_name = os.getenv('DXTRADE_ACCOUNT_NAME', 'demo') account = f"{domain}:{account_name}" - + try: if channel in ["quotes", "market_data"]: # Market Data Subscription Request subscription_message = { "type": "MarketDataSubscriptionRequest", "requestId": request_id, + "timestamp": _utc_timestamp(), "session": session_token, "payload": { "account": account, @@ -733,28 +758,29 @@ async def _send_dxtrade_subscription(self, websocket: websockets.WebSocketClient } } else: - # Account/Portfolio Subscription Request + # Account/Portfolio Subscription Request subscription_message = { - "type": "AccountPortfoliosSubscriptionRequest", + "type": "AccountPortfoliosSubscriptionRequest", "requestId": request_id, + "timestamp": _utc_timestamp(), "session": session_token, "payload": { - "account": account, - "eventTypes": [{"type": "Position", "format": "COMPACT"}] + "requestType": "LIST", + "accounts": [account] } } - + logger.info(f"📡 Sending DXTrade subscription for {channel}: {account}") logger.debug(f"Subscription message: {subscription_message}") - + await websocket.send(json.dumps(subscription_message)) - + except Exception as e: logger.error(f"Error sending DXTrade subscription for {channel}: {e}") # Fallback to simple subscription fallback_message = {"type": "subscribe", "channel": channel} await websocket.send(json.dumps(fallback_message)) - + async def send_market_data_subscription(self, symbols: list, account: Optional[str] = None, event_types: Optional[list] = None) -> Optional[dict]: """Send market data subscription with DXTrade format. @@ -767,12 +793,12 @@ async def send_market_data_subscription(self, symbols: list, account: Optional[s Response message if any """ import uuid - + # Get session token token = self.auth_handler.get_session_token() if not token: raise ValueError("No session token available") - + # Use provided account or get from config if not account: account = getattr(self.config, 'account', None) @@ -780,15 +806,16 @@ async def send_market_data_subscription(self, symbols: list, account: Optional[s domain = getattr(self.config.auth, 'domain', 'default') account_name = os.getenv('DXTRADE_ACCOUNT_NAME', 'demo') account = f"{domain}:{account_name}" - + # Default event types if not event_types: event_types = [{"type": "Quote", "format": "COMPACT"}] - + # Create subscription message subscription_message = { "type": "MarketDataSubscriptionRequest", "requestId": str(uuid.uuid4()), + "timestamp": _utc_timestamp(), "session": token, "payload": { "account": account, @@ -796,34 +823,37 @@ async def send_market_data_subscription(self, symbols: list, account: Optional[s "eventTypes": event_types } } - + # Send via quotes channel websocket = self._websockets.get("quotes") if not websocket: raise ValueError("Not connected to market data channel") - + await websocket.send(json.dumps(subscription_message)) logger.info(f"📡 Sent market data subscription: {symbols} on account {account}") - + return None - - async def send_portfolio_subscription(self, account: Optional[str] = None, event_types: Optional[list] = None) -> Optional[dict]: + + async def send_portfolio_subscription(self, account: Optional[str] = None) -> Optional[dict]: """Send portfolio subscription with DXTrade format. - + + The DXTrade Push API expects the account portfolios subscription + payload to list accounts (``requestType``/``accounts``); see the + DXtrade Push API specification. + Args: account: Account identifier (defaults to config account) - event_types: Event types to subscribe to (defaults to Position COMPACT) - + Returns: Response message if any """ import uuid - + # Get session token token = self.auth_handler.get_session_token() if not token: raise ValueError("No session token available") - + # Use provided account or get from config if not account: account = getattr(self.config, 'account', None) @@ -831,69 +861,58 @@ async def send_portfolio_subscription(self, account: Optional[str] = None, event domain = getattr(self.config.auth, 'domain', 'default') account_name = os.getenv('DXTRADE_ACCOUNT_NAME', 'demo') account = f"{domain}:{account_name}" - - # Default event types - if not event_types: - event_types = [{"type": "Position", "format": "COMPACT"}] - + # Create subscription message subscription_message = { "type": "AccountPortfoliosSubscriptionRequest", "requestId": str(uuid.uuid4()), + "timestamp": _utc_timestamp(), "session": token, "payload": { - "account": account, - "eventTypes": event_types - } + "requestType": "LIST", + "accounts": [account], + }, } - + # Send via portfolio channel websocket = self._websockets.get("portfolio") if not websocket: raise ValueError("Not connected to portfolio channel") - + await websocket.send(json.dumps(subscription_message)) logger.info(f"📡 Sent portfolio subscription on account {account}") - + return None async def send_message(self, channel: str, message: Union[dict, str]) -> Optional[dict]: """Send raw message to WebSocket channel. - + + Incoming messages are delivered to the channel's callback by the + background message handler, so sending never blocks on a reply. + Args: channel: Channel name message: Message to send (dict will be JSON encoded) - + Returns: - Response message if any + None (replies arrive via the channel callback) """ websocket = self._websockets.get(channel) if not websocket: raise ValueError(f"Not connected to channel: {channel}") - + # Encode message if needed if isinstance(message, dict): message = json.dumps(message) - + await websocket.send(message) - - # Wait for response (optional - might want to handle differently) - try: - response = await asyncio.wait_for(websocket.recv(), timeout=5.0) - if isinstance(response, str): - try: - return json.loads(response) - except json.JSONDecodeError: - return response - return response - except asyncio.TimeoutError: - return None - + return None + # Convenience methods for common operations async def get_accounts(self) -> Union[Dict, list]: """Get accounts (raw data).""" return await self.request("GET", "/accounts") - + async def get_orders(self, account_id: Optional[str] = None) -> Union[Dict, list]: """Get orders (raw data).""" endpoint = "/orders" @@ -901,11 +920,11 @@ async def get_orders(self, account_id: Optional[str] = None) -> Union[Dict, list if account_id: params['account_id'] = account_id return await self.request("GET", endpoint, params=params) - + async def create_order(self, order_data: dict) -> dict: """Create order (raw data).""" return await self.request("POST", "/orders", json=order_data) - + async def get_positions(self, account_id: Optional[str] = None) -> Union[Dict, list]: """Get positions (raw data).""" endpoint = "/positions" @@ -913,7 +932,7 @@ async def get_positions(self, account_id: Optional[str] = None) -> Union[Dict, l if account_id: params['account_id'] = account_id return await self.request("GET", endpoint, params=params) - + async def get_quotes(self, symbols: Optional[list] = None) -> Union[Dict, list]: """Get quotes (raw data).""" endpoint = "/quotes" @@ -921,11 +940,237 @@ async def get_quotes(self, symbols: Optional[list] = None) -> Union[Dict, list]: if symbols: params['symbols'] = ','.join(symbols) return await self.request("GET", endpoint, params=params) - + async def get_server_time(self) -> Union[Dict, str]: """Get server time (raw data).""" return await self.request("GET", "/time") + # ------------------------------------------------------------------ + # DXTrade REST API methods (per the official OpenAPI specification) + # ------------------------------------------------------------------ + # These methods follow the DXTrade REST API resource layout shared by + # all DXTrade brokers (accounts are addressed by their full code, e.g. + # "default:12345", percent-encoded in the path). + + @staticmethod + def _encode_account(account: str) -> str: + """Percent-encode an account code for use in a REST path. + + DXTrade account codes contain a colon (``default:12345``); the + colon must be percent-encoded (``default%3A12345``) when the code + is placed in a path segment. + + Args: + account: Full account code (e.g. ``default:12345``) + + Returns: + Percent-encoded account code + """ + return quote(account, safe="") + + async def get_users(self) -> Union[Dict[str, Any], list, str]: + """Get users and the accounts they can access. + + Account discovery: the response contains the full account codes + (e.g. ``default:12345``) used to address account-scoped resources. + + Returns: + Raw users/accounts response + """ + return await self.request("GET", "/users") + + async def get_account_metrics(self, account: str) -> Union[Dict[str, Any], list, str]: + """Get account metrics (equity, balance, margin, PnL). + + Args: + account: Full account code (e.g. ``default:12345``) + + Returns: + Raw account metrics response + """ + return await self.request( + "GET", f"/accounts/{self._encode_account(account)}/metrics" + ) + + async def get_account_portfolio(self, account: str) -> Union[Dict[str, Any], list, str]: + """Get the account portfolio (open positions and working orders). + + Args: + account: Full account code (e.g. ``default:12345``) + + Returns: + Raw account portfolio response + """ + return await self.request( + "GET", f"/accounts/{self._encode_account(account)}/portfolio" + ) + + async def get_account_positions(self, account: str) -> Union[Dict[str, Any], list, str]: + """Get open positions for an account. + + Args: + account: Full account code (e.g. ``default:12345``) + + Returns: + Raw positions response + """ + return await self.request( + "GET", f"/accounts/{self._encode_account(account)}/positions" + ) + + async def get_account_orders(self, account: str) -> Union[Dict[str, Any], list, str]: + """Get orders for an account. + + Args: + account: Full account code (e.g. ``default:12345``) + + Returns: + Raw orders response + """ + return await self.request( + "GET", f"/accounts/{self._encode_account(account)}/orders" + ) + + async def get_account_orders_history(self, account: str) -> Union[Dict[str, Any], list, str]: + """Get order history for an account. + + Args: + account: Full account code (e.g. ``default:12345``) + + Returns: + Raw order history response + """ + return await self.request( + "GET", f"/accounts/{self._encode_account(account)}/orders/history" + ) + + async def query_instruments( + self, + symbols: Optional[list] = None, + account: Optional[str] = None, + limit: Optional[int] = None, + ) -> Union[Dict[str, Any], list, str]: + """Query available instruments. + + Args: + symbols: Optional list of symbols to filter by + account: Optional full account code (e.g. ``default:12345``). + Account-scoped discovery is recommended because instrument + availability is account-specific. + limit: Optional maximum number of results + + Returns: + Raw instruments response + """ + params: Dict[str, Any] = {} + if symbols: + params['symbols'] = ','.join(symbols) + if limit is not None: + params['limit'] = limit + + if account: + endpoint = f"/accounts/{self._encode_account(account)}/instruments/query" + else: + endpoint = "/instruments/query" + return await self.request("GET", endpoint, params=params) + + async def get_market_data( + self, + symbols: list, + event_types: Optional[list] = None, + account: Optional[str] = None, + ) -> Union[Dict[str, Any], list, str]: + """Request a market data snapshot over REST. + + Args: + symbols: List of symbols to request data for + event_types: Event types to request (defaults to + ``[{"type": "Quote", "format": "COMPACT"}]``) + account: Optional full account code (e.g. ``default:12345``) + + Returns: + Raw market data response + """ + if not event_types: + event_types = [{"type": "Quote", "format": "COMPACT"}] + payload: Dict[str, Any] = {"symbols": symbols, "eventTypes": event_types} + if account: + payload["account"] = account + return await self.request("POST", "/marketdata", json=payload) + + async def ping(self) -> Union[Dict[str, Any], list, str]: + """Validate the session and refresh the token if the server returns one. + + The server may return a fresh ``sessionToken``; when present the + stored token is updated so subsequent requests stay authenticated. + + Returns: + Raw ping response + """ + response = await self.request("POST", "/ping") + if isinstance(response, dict): + new_token = response.get("sessionToken") + if new_token: + self.auth_handler._session_token = new_token + self.auth_handler._token_expires_at = time.time() + 3600 + self.auth_handler._last_login = time.time() + return response + + async def logout(self) -> Union[Dict[str, Any], list, str]: + """Invalidate the session on the server and clear the local token. + + Returns: + Raw logout response + """ + try: + return await self.request("POST", "/logout") + finally: + self.auth_handler._session_token = None + self.auth_handler._token_expires_at = None + self.auth_handler._last_login = None + + async def place_order(self, account: str, order: Dict[str, Any]) -> Union[Dict[str, Any], list, str]: + """Place an order on an account. + + ``order`` follows the DXTrade ``SingleOrderRequest`` schema: at minimum + ``orderCode`` (client-generated, unique per account), ``type`` + (``MARKET``/``LIMIT``/``STOP``), ``instrument``, ``side`` (``BUY``/ + ``SELL``), and ``tif`` (e.g. ``GTC``). For closing positions set + ``positionEffect: "CLOSE"`` plus the ``positionCode`` and the opposite + ``side``; omit ``quantity`` to close the full position. + + A ``200`` response is an acknowledgement, not proof of execution — + confirm fills via ``get_account_orders``/``get_account_positions``. + + Args: + account: Full account code (e.g. ``default:12345``) + order: SingleOrderRequest fields + + Returns: + Raw order response + """ + payload = dict(order) + payload.setdefault("account", account) + return await self.request( + "POST", f"/accounts/{self._encode_account(account)}/orders", json=payload + ) + + async def cancel_order(self, account: str, order: str) -> Union[Dict[str, Any], list, str]: + """Cancel a working order. + + Args: + account: Full account code (e.g. ``default:12345``) + order: Order code — either the client order code or the system + order id + + Returns: + Raw cancel response + """ + encoded_order = quote(order, safe="") + return await self.request( + "DELETE", f"/accounts/{self._encode_account(account)}/orders/{encoded_order}" + ) + # Convenience factory function def create_transport(config=None) -> DXTradeTransport: @@ -937,4 +1182,4 @@ def create_transport(config=None) -> DXTradeTransport: Returns: DXTradeTransport client """ - return DXTradeTransport(config) \ No newline at end of file + return DXTradeTransport(config) diff --git a/src/dxtrade/utils.py b/src/dxtrade/utils.py new file mode 100644 index 0000000..23da44a --- /dev/null +++ b/src/dxtrade/utils.py @@ -0,0 +1,609 @@ +"""High-level helpers built on the DXTrade transport layer. + +These utilities wrap the raw transport methods with the common orchestration +needed to stream market data and open/close positions on any DXTrade broker: +account discovery, symbol resolution, client order codes, protective stops +and flat-state verification. They trade convenience for control — use the +transport directly when you need full control. +""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from collections.abc import Callable +from typing import Any + +from .transport import DXTradeTransport + +logger = logging.getLogger(__name__) + +#: Order statuses that are no longer active. +FINAL_STATUSES = ("COMPLETED", "CANCELED", "EXPIRED", "REJECTED") + +#: Default order time-in-force for market orders. +DEFAULT_TIF = "GTC" + + +def order_code(prefix: str = "vt") -> str: + """Generate a unique client order code. + + The DXTrade API requires ``orderCode`` to be client-generated and unique + per account. + + Args: + prefix: Short prefix for the generated code + + Returns: + Unique order code, e.g. ``vt-3f2a9c1d0b4e`` + """ + return f"{prefix}-{uuid.uuid4().hex[:12]}" + + +def find_account_code(users: Any) -> str: + """Extract the first account code from a ``/users`` response. + + DXTrade account codes look like ``default:12345``. The exact ``/users`` + response shape varies between brokers, so several shapes are handled. + + Args: + users: Raw ``/users`` response (dict or list) + + Returns: + First account code found (e.g. ``default:12345``) + + Raises: + ValueError: No account code could be located + """ + + def find_in(item: Any) -> str: + if isinstance(item, dict): + for key in ("accountCode", "account", "id"): + value = item.get(key) + if isinstance(value, str) and value.startswith("default:"): + return value + for key in ("accounts", "users", "userDetails", "accountList"): + value = item.get(key) + if isinstance(value, list): + for entry in value: + found = find_in(entry) + if found: + return found + elif isinstance(item, list): + for entry in item: + found = find_in(entry) + if found: + return found + return "" + + code = find_in(users) + if not code: + raise ValueError("Could not find an account code in /users response") + return code + + +def as_list(payload: Any, key: str) -> list[dict[str, Any]]: + """Normalise a positions/orders/metrics response into a list of dicts. + + The DXTrade REST API returns these resources either as a bare list or as + ``{key: [...]}``; this helper handles both. + + Args: + payload: Raw response (dict or list) + key: Expected list key (e.g. ``positions``, ``orders``, ``metrics``) + + Returns: + List of dict entries + """ + if isinstance(payload, list): + return [item for item in payload if isinstance(item, dict)] + if isinstance(payload, dict): + items = payload.get(key, []) + if isinstance(items, list): + return [item for item in items if isinstance(item, dict)] + return [] + + +def instrument_items(instruments: Any) -> list[dict[str, Any]]: + """Extract instrument records from an instruments query response.""" + if isinstance(instruments, dict): + for key in ("instrumentDetails", "instruments", "symbols"): + value = instruments.get(key) + if isinstance(value, list): + return [i for i in value if isinstance(i, dict)] + for _key, value in instruments.items(): + if isinstance(value, list) and value and isinstance(value[0], dict): + return [i for i in value if isinstance(i, dict)] + elif isinstance(instruments, list): + return [i for i in instruments if isinstance(i, dict)] + return [] + + +def find_instrument(instruments: Any, symbol: str) -> dict[str, Any]: + """Find the instrument record for a symbol in an instruments response. + + Args: + instruments: Raw instruments query response + symbol: Exact symbol to find + + Returns: + Instrument record + + Raises: + ValueError: Instrument not present in the response + """ + for item in instrument_items(instruments): + if item.get("symbol") == symbol: + return item + raise ValueError(f"instrument {symbol!r} not in query response") + + +def _match_symbol(hint: str, symbols: list[str]) -> str | None: + """Match a user symbol hint against exact platform symbols. + + Handles common naming differences, e.g. ``BTCUSDT`` -> ``BTCUSD``. + + Args: + hint: User-provided symbol hint + symbols: Platform symbols to match against + + Returns: + Matching platform symbol, or None + """ + hint_upper = hint.upper() + for symbol in symbols: + if symbol.upper() == hint_upper: + return symbol + # BTCUSDT vs BTCUSD style one-letter quote mismatch + if hint_upper.endswith("T"): + for symbol in symbols: + if symbol.upper() == hint_upper[:-1]: + return symbol + for symbol in symbols: + symbol_upper = symbol.upper() + if hint_upper in symbol_upper or symbol_upper in hint_upper: + return symbol + return None + + +async def resolve_account(transport: DXTradeTransport) -> str: + """Ensure authentication and return the first accessible account code. + + Args: + transport: Authenticated transport (logs in first if needed) + + Returns: + Full account code (e.g. ``default:12345``) + """ + if not transport.auth_handler.get_session_token(): + await transport.authenticate() + users = await transport.get_users() + return find_account_code(users) + + +async def resolve_symbol(transport: DXTradeTransport, account: str, hint: str) -> str: + """Resolve a symbol hint to the exact platform symbol. + + Queries the account's instruments with the hint and common variants + (e.g. ``BTCUSDT`` also queries ``BTCUSD``, ``BTC/USD`` also queries + ``BTCUSD``). + + Args: + transport: Authenticated transport + account: Full account code + hint: User-provided symbol hint + + Returns: + Exact platform symbol + + Raises: + ValueError: The symbol could not be resolved + """ + candidates = [hint] + upper = hint.upper() + if upper.endswith("USDT"): + candidates.append(hint[:-1]) + if "/" in hint: + candidates.append(hint.replace("/", "")) + candidates = list(dict.fromkeys(candidates)) # de-duplicate, keep order + + last_error: Exception | None = None + for candidate in candidates: + try: + instruments = await transport.query_instruments( + symbols=[candidate], account=account + ) + except Exception as exc: + last_error = exc + continue + matched = _match_symbol( + hint, [item.get("symbol", "") for item in instrument_items(instruments)] + ) + if matched: + return matched + last_error = ValueError(f"no match for {candidate!r}") + + raise ValueError( + f"could not resolve symbol {hint!r} on account {account}: {last_error}" + ) + + +async def discover_symbols( + transport: DXTradeTransport, + account: str, + limit: int = 5, + query_limit: int = 50, +) -> list[str]: + """Discover a few tradable symbols for an account. + + Args: + transport: Authenticated transport + account: Full account code + limit: Maximum number of symbols to return + query_limit: Maximum instruments to fetch + + Returns: + List of platform symbols + """ + instruments = await transport.query_instruments(account=account, limit=query_limit) + symbols = [ + item.get("symbol", "") + for item in instrument_items(instruments) + if item.get("symbol") + ] + return symbols[:limit] + + +def _quote(market_data: Any) -> tuple[float | None, float | None]: + """Extract (bid, ask) from a POST /marketdata response.""" + events = market_data.get("events", []) if isinstance(market_data, dict) else [] + for event in events: + if isinstance(event, dict) and event.get("type") == "Quote": + return event.get("bid"), event.get("ask") + return None, None + + +async def wait_for_position( + transport: DXTradeTransport, + account: str, + symbol: str | None = None, + timeout: float = 30.0, +) -> dict[str, Any] | None: + """Wait for a position to appear on the account. + + Args: + transport: Authenticated transport + account: Full account code + symbol: Optional symbol filter + timeout: Maximum wait in seconds + + Returns: + The position record, or None on timeout + """ + loop = asyncio.get_event_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + positions = as_list(await transport.get_account_positions(account), "positions") + match = next( + (p for p in positions if not symbol or p.get("symbol") == symbol), + None, + ) + if match: + return match + await asyncio.sleep(0.5) + return None + + +async def wait_no_position( + transport: DXTradeTransport, + account: str, + position_code: str, + timeout: float = 15.0, +) -> bool: + """Wait until a specific position is gone. + + Args: + transport: Authenticated transport + account: Full account code + position_code: Position code to watch + timeout: Maximum wait in seconds + + Returns: + True if the position disappeared, False on timeout + """ + loop = asyncio.get_event_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + positions = as_list(await transport.get_account_positions(account), "positions") + if not any(str(p.get("positionCode")) == str(position_code) for p in positions): + return True + await asyncio.sleep(0.5) + return False + + +async def stream_quotes( + transport: DXTradeTransport, + symbols: list[str] | None = None, + account: str | None = None, + duration: float = 30.0, + on_quote: Callable[[dict[str, Any]], None] | None = None, +) -> list[dict[str, Any]]: + """Stream real-time quotes for a duration. + + Symbols default to a small set discovered for the account. Quote events + (each a dict with ``symbol``/``bid``/``ask``/``time``) are collected and + returned; ``on_quote`` is also called per event when provided. + + Args: + transport: Transport (authenticated on demand) + symbols: Optional symbol hints to stream + account: Optional full account code (discovered if omitted) + duration: How long to stream, in seconds + on_quote: Optional callback per quote event + + Returns: + List of quote events received during the stream + + Raises: + TimeoutError: The market data channel did not connect + """ + account = account or await resolve_account(transport) + if symbols: + symbols = [await resolve_symbol(transport, account, s) for s in symbols] + else: + symbols = await discover_symbols(transport, account, limit=5) + if not symbols: + raise ValueError("no symbols to stream") + + received: list[dict[str, Any]] = [] + + def callback(message: Any) -> None: + if isinstance(message, dict) and message.get("type") == "MarketData": + for event in message.get("payload", {}).get("events", []): + if isinstance(event, dict): + received.append(event) + if on_quote: + on_quote(event) + + await transport.subscribe("quotes", callback) + connected = await transport.wait_for_channel("quotes", timeout=30.0) + if not connected: + await transport.unsubscribe("quotes") + raise TimeoutError("market data channel did not connect") + try: + await transport.send_market_data_subscription(symbols, account) + await asyncio.sleep(duration) + finally: + await transport.unsubscribe("quotes") + + logger.info(f"stream_quotes: {len(received)} quote events for {symbols}") + return received + + +def _close_side(side: str) -> str: + """Return the closing side opposite to an open side.""" + return "SELL" if side == "BUY" else "BUY" + + +async def open_position( + transport: DXTradeTransport, + symbol: str, + side: str = "BUY", + quantity: float | None = None, + stop_loss: float | None = None, + stop_loss_price: float | None = None, + account: str | None = None, + fill_timeout: float = 30.0, +) -> dict[str, Any]: + """Open a market position, optionally with a protective stop loss. + + Quantity defaults to the instrument's minimum order size. ``stop_loss`` + is a maximum loss in account currency (converted to a price using the + current quote); ``stop_loss_price`` is an absolute price level and takes + precedence when both are given. + + The protective stop is placed as a closing STOP order without a quantity, + which is what the DXTrade API requires for closing STOP/LIMIT orders. + + Args: + transport: Transport (authenticated on demand) + symbol: Symbol hint (e.g. ``BTCUSDT``) + side: ``BUY`` or ``SELL`` + quantity: Position size in base units (defaults to minimum) + stop_loss: Optional max loss in account currency + stop_loss_price: Optional absolute stop price + account: Optional full account code (discovered if omitted) + fill_timeout: Max seconds to wait for the position to fill + + Returns: + Dict with ``order`` (placement reply), ``position`` (filled + position record) and ``stop_order`` (stop placement reply or None) + + Raises: + RuntimeError: The order did not fill + ValueError: Invalid arguments or unresolvable symbol + """ + account = account or await resolve_account(transport) + symbol = await resolve_symbol(transport, account, symbol) + side = side.upper() + if side not in ("BUY", "SELL"): + raise ValueError(f"side must be BUY or SELL, got {side!r}") + + instruments = await transport.query_instruments(symbols=[symbol], account=account) + instrument = find_instrument(instruments, symbol) + if quantity is None: + quantity = instrument.get("minOrderSize") + if not quantity: + raise ValueError("no minOrderSize for instrument and no quantity given") + + market_data = await transport.get_market_data([symbol], account=account) + bid, ask = _quote(market_data) + reference = ask if side == "BUY" else bid + if not reference: + raise RuntimeError("no quote received to size the order") + + stop_price = stop_loss_price + if stop_loss is not None and stop_price is None: + delta = stop_loss / quantity + stop_price = reference - delta if side == "BUY" else reference + delta + stop_price = round(stop_price, 2) + + open_order = { + "orderCode": order_code(), + "type": "MARKET", + "instrument": symbol, + "quantity": quantity, + "positionEffect": "OPEN", + "side": side, + "tif": DEFAULT_TIF, + } + placed = await transport.place_order(account, open_order) + + position = await wait_for_position(transport, account, symbol, timeout=fill_timeout) + if not position: + raise RuntimeError(f"order {placed} did not fill: no position appeared") + + result: dict[str, Any] = { + "order": placed, + "position": position, + "stop_order": None, + } + if stop_price is not None: + stop_order = { + "orderCode": order_code("vt-stop"), + "type": "STOP", + "instrument": symbol, + "positionEffect": "CLOSE", + "positionCode": position["positionCode"], + "side": _close_side(side), + "stopPrice": stop_price, + "tif": DEFAULT_TIF, + } + result["stop_order"] = await transport.place_order(account, stop_order) + return result + + +async def close_position( + transport: DXTradeTransport, + account: str | None = None, + symbol: str | None = None, + position_code: str | None = None, +) -> dict[str, Any]: + """Close a position with a market order (full close). + + Exactly one open position must match: either ``position_code``, or + ``symbol`` (or the only open position when neither is given). + + Args: + transport: Transport (authenticated on demand) + account: Optional full account code (discovered if omitted) + symbol: Optional symbol filter + position_code: Optional position code to close + + Returns: + Dict with ``order`` (placement reply) and ``position`` (the record + that was closed) + + Raises: + ValueError: Zero or multiple matching positions + """ + account = account or await resolve_account(transport) + positions = as_list(await transport.get_account_positions(account), "positions") + if position_code: + matches = [ + p for p in positions if str(p.get("positionCode")) == str(position_code) + ] + else: + matches = [p for p in positions if not symbol or p.get("symbol") == symbol] + if len(matches) != 1: + raise ValueError( + f"expected exactly one open position to close, found {len(matches)}" + ) + + position = matches[0] + position_code = position["positionCode"] + symbol = position["symbol"] + close_order = { + "orderCode": order_code("vt-close"), + "type": "MARKET", + "instrument": symbol, + "positionEffect": "CLOSE", + "positionCode": position_code, + "side": _close_side(position.get("side", "BUY")), + "tif": DEFAULT_TIF, + } + placed = await transport.place_order(account, close_order) + await wait_no_position(transport, account, str(position_code), timeout=15.0) + return {"order": placed, "position": position} + + +async def flatten( + transport: DXTradeTransport, account: str | None = None +) -> dict[str, Any]: + """Close all open positions and cancel all working orders. + + Args: + transport: Transport (authenticated on demand) + account: Optional full account code (discovered if omitted) + + Returns: + Dict with ``closed`` (position codes), ``cancelled`` (order codes) + and ``errors`` (any failures) + """ + account = account or await resolve_account(transport) + result: dict[str, Any] = {"closed": [], "cancelled": [], "errors": []} + + positions = as_list(await transport.get_account_positions(account), "positions") + for position in positions: + close_order = { + "orderCode": order_code("vt-close"), + "type": "MARKET", + "instrument": position["symbol"], + "positionEffect": "CLOSE", + "positionCode": position["positionCode"], + "side": _close_side(position.get("side", "BUY")), + "tif": DEFAULT_TIF, + } + try: + await transport.place_order(account, close_order) + result["closed"].append(position["positionCode"]) + except Exception as exc: + result["errors"].append(f"close {position['positionCode']}: {exc!r}") + + await asyncio.sleep(1) + orders = as_list(await transport.get_account_orders(account), "orders") + for order in orders: + if order.get("status") in FINAL_STATUSES: + continue + ref = order.get("orderCode") or order.get("orderId") + try: + await transport.cancel_order(account, str(ref)) + result["cancelled"].append(ref) + except Exception as exc: + result["errors"].append(f"cancel {ref}: {exc!r}") + + return result + + +async def account_is_flat( + transport: DXTradeTransport, account: str | None = None +) -> bool: + """Check whether the account has no open positions or working orders. + + Args: + transport: Transport (authenticated on demand) + account: Optional full account code (discovered if omitted) + + Returns: + True when ``openPositionsCount`` and ``openOrdersCount`` are both 0 + """ + account = account or await resolve_account(transport) + metrics = as_list(await transport.get_account_metrics(account), "metrics") + if not metrics: + return False + first = metrics[0] + return ( + int(first.get("openPositionsCount") or 0) == 0 + and int(first.get("openOrdersCount") or 0) == 0 + ) diff --git a/tests.md b/tests.md new file mode 100644 index 0000000..883ac00 --- /dev/null +++ b/tests.md @@ -0,0 +1,212 @@ +# Velotrade Integration Test Results + +Live results from testing `dxtrade-python-sdk` against **Velotrade's DXtrade +platform** (`dx.velotrade.com`). Test date: **2026-08-14**. + +**Summary: 60/60 tests pass** — 52 offline (mocked) + 8 live (real Velotrade +account). All live tests were read-only and left the account flat. + +--- + +## 1. Test suite layout + +| File | Tests | Kind | +|---|---|---| +| `tests/test_auth.py` | 29 | Offline unit — auth handlers incl. new `get_auth_headers()` | +| `tests/test_transport.py` | 25 | Offline unit — REST methods, order placement/cancel, subscription shapes, ping/pong, `send_message` | +| `tests/test_utils.py` | 19 | Offline unit — high-level utils: streaming, open/close position, flatten, symbol/account resolution | +| `tests/test_velotrade_live.py` | 4 | Live — login, authenticated REST, full users→instruments flow, ping/logout | +| `tests/test_velotrade_ws_live.py` | 4 | Live — Push API: quotes stream, portfolio snapshot, legacy-payload rejection, ping stats | +| **Total** | **81** | | + +Run everything (offline + live, requires broker `.env`): + +```bash +venv/Scripts/python.exe -m pytest tests/ -q --no-cov +``` + +Live tests are skipped automatically when `.env` with `DXTRADE_USERNAME` is +absent, so CI stays hermetic. + +--- + +## 2. Live test results (Velotrade) + +### 2.1 REST — `tests/test_velotrade_live.py` ✅ 4/4 + +| Test | Result | Evidence | +|---|---|---| +| `test_authenticate_returns_session_token` | ✅ PASS | `POST /dxsca-web/login` → `sessionToken` returned | +| `test_authenticated_request_uses_token` | ✅ PASS | `GET /dxsca-web/users` → 200 with `Authorization: DXAPI ` | +| `test_full_flow_users_and_instruments` | ✅ PASS | login → `/users` → account `default:130000606` → account-scoped instruments query | +| `test_ping_and_logout` | ✅ PASS | `POST /ping` OK; `POST /logout` OK; local token cleared | + +### 2.2 Push API (WebSocket) — `tests/test_velotrade_ws_live.py` ✅ 4/4 + +| Test | Result | Evidence | +|---|---|---| +| `test_market_data_subscription_streams_quotes` | ✅ PASS | Real-time `MarketData` events with `bid`/`ask` for requested symbols | +| `test_portfolio_subscription_receives_snapshot` | ✅ PASS | `AccountPortfolios` snapshot received with `payload.portfolios` | +| `test_portfolio_legacy_payload_rejected` | ✅ PASS | Old SDK payload shape → server **rejects** (see §3.2) | +| `test_ping_stats_tracked_on_active_channels` | ✅ PASS | Stats structures present; session health 1.0 | + +--- + +## 3. Real message bodies captured from Velotrade + +Captured with `tests/_diag_push.py` (diagnostic; not a pytest test). + +### 3.1 Login & discovery + +``` +POST /dxsca-web/login {username, password, domain:"default"} -> {sessionToken: ""} +GET /dxsca-web/users -> account code: default:130000606 +GET /dxsca-web/accounts/default%3A130000606/instruments/query?limit=100 + -> symbols: ['AAOI', 'AAPL', 'AAVEUSD', 'ADAUSD', 'ADBE', 'AEROUSD', 'ALGOUSD', 'AMAT', ...] +``` + +### 3.2 Portfolio subscription — before/after the payload fix (gap G5) + +Legacy SDK payload (rejected): + +```json +{"type":"AccountPortfoliosSubscriptionRequest","requestId":"diag-legacy","session":"…", + "payload":{"account":"default:130000606","eventTypes":[{"type":"Position","format":"COMPACT"}]}} +``` + +Server reply: + +```json +{"type":"Reject","inReplyTo":"diag-legacy","session":"…", + "payload":{"errorCode":"32","description":"Incorrect request parameters: "}} +``` + +Spec payload (accepted — what the SDK sends now): + +```json +{"type":"AccountPortfoliosSubscriptionRequest","requestId":"…","timestamp":"…","session":"…", + "payload":{"requestType":"LIST","accounts":["default:130000606"]}} +``` + +Server reply: + +```json +{"type":"AccountPortfolios","inReplyTo":"…","session":"…","timestamp":"…", + "payload":{"portfolios":[{"account":"default:130000606","version":18, + "balances":[{"account":"default:130000606","version":18,"value":5000.0,"currency":"USD"}], + "positions":[],"orders":[],"owner":{"login":"3807193346…"}}]}} +``` + +### 3.3 Market data stream + +```json +{"type":"MarketData","inReplyTo":"…","session":"…","timestamp":"2026-08-14T19:30:35.300Z", + "payload":{"events":[ + {"symbol":"AAVEUSD","type":"Quote","ask":85.91,"bid":85.89,"time":"2026-08-14T19:30:35Z"}, + {"symbol":"ADBE","type":"Quote","ask":265.59,"bid":264.67,"time":"2026-08-14T19:30:35Z"}, + {"symbol":"ADAUSD","type":"Quote","ask":0.17897,"bid":0.17896,"time":"2026-08-14T19:30:35Z"}]}} +``` + +### 3.4 Ping/pong & session health (observed over a longer session) + +``` +ping stats (quotes): {"ping_requests_received": 2, "ping_responses_sent": 2, + "session_extensions": 2, ...} +session health: {"active_channels": 2, "healthy_channels": 2, "session_health": 1.0, + "ping_response_success_rate": 1.0, + "connection_strategies": {"portfolio": "additional_headers", + "quotes": "additional_headers"}, + "websockets_version": "17.0.1"} +``` + +The SDK's automatic `Ping` reply to the server's `PingRequest` works live, and +both sockets connected via the `additional_headers` strategy. + +### 3.5 Explicit subscription close (Push spec) + +```json +{"type":"AccountPortfoliosCloseSubscriptionRequest","requestId":"diag-close", + "refRequestId":"","session":"…","timestamp":"…"} +``` + +Server reply: + +```json +{"type":"AccountPortfoliosSubscriptionClosed","inReplyTo":"diag-close","session":"…","timestamp":"…"} +``` + +--- + +## 4. SDK changes made to pass these tests + +All changes are broker-agnostic — they follow the official DXTrade API +specifications (REST OpenAPI + Push API), which any DXTrade broker shares. + +| Change | File(s) | Why | +|---|---|---| +| `get_auth_headers()` on all auth handlers; transport builds headers through the handler | `auth.py`, `transport.py` | Velotrade requires `Authorization: DXAPI `; SDK only sent `X-Auth-Token` → 401 on `/users` | +| New REST methods: `get_users`, `get_account_metrics/portfolio/positions/orders/orders_history`, `query_instruments`, `get_market_data`, `ping`, `logout`, `_encode_account` | `transport.py` | SDK's old convenience methods hit non-existent paths (`/orders`, `/positions`, `/time`…) → 404 | +| Portfolio subscription payload → `{requestType:"LIST", accounts:[…]}` | `transport.py` | Old payload rejected by server: `errorCode 32 ` | +| `timestamp` added to all Push subscription messages | `transport.py` | Required by the Push API message envelope | +| `send_message()` no longer calls `recv()` | `transport.py` | `recv` collided with the background handler → `websockets.ConcurrencyError` | +| Ping stats stored via `setdefault` | `transport.py` | Counters were dropped before the channel's stats dict existed | +| `env_config.py` reads `DXTRADE_WS_MARKET_DATA_URL` / `DXTRADE_WS_PORTFOLIO_URL` | `env_config.py` | Documented vars were never loaded; `subscribe()` without explicit URL failed | +| `httpx` declared in `pyproject.toml` | `pyproject.toml` | Imported by `auth.py` but undeclared | + +--- + +## 5. Live trade execution (2026-08-14, account `default:130000606`) + +### 6.1 Intended trade — BTCUSD buy + $10 stop loss, closed after 30 s + +| Step | Order | Details | +|---|---|---| +| Open | 2980544 MARKET BUY | 0.001 BTCUSD @ 62,956.7 (platform min, ~$63 notional) | +| Stop | 2980550 STOP SELL | protective stop @ 52,956.7 (= entry − 10,000 pts = $10 loss on 0.001 BTC) | +| Hold | — | 30 seconds | +| Close | 2980555 MARKET SELL | full close; position gone, `openOrdersCount=0` | + +Net result: **flat**, equity $4,999.77, session PnL −$0.03 (spread only). + +### 6.2 Findings from the trade run + +- **Symbol naming**: Velotrade uses `BTCUSD` (crypto/FOREX type, min order + 0.001, margin rate 0.1667); `BTCUSDT` is not a valid symbol. The utils' + `resolve_symbol()` handles `BTCUSDT` → `BTCUSD` automatically. +- **Closing STOP/LIMIT orders must NOT carry a quantity** — the server rejects + them with `errorCode 33 "Incorrect request. Closing STOP/LIMIT orders should + not have specified quantity"`. The protective stop is placed without + `quantity` (full close). +- **Protective stops auto-cancel** when their parent position closes — no + orphaned working orders remain (confirmed via `/orders` + `openOrdersCount`). +- **Working-order detection**: the orders list does not reliably echo the stop's + `orderCode`; `metrics.openOrdersCount` is the reliable signal (the utils' + `account_is_flat()` uses it). +- **Order/position codes** on this deployment are numeric strings (e.g. + `2980544`); `positionCode` from `/positions` is used directly in closing and + stop orders. +- **`/orders` history** uses `orderCode` prefixed with + `dxsca-integration-session-code:` — client codes remain unique per account. + +### 6.3 Diagnostic scripts + +- `tests/_diag_trade.py` — the full open/stop/hold/close/verify flow with + `--dry-run` support and guaranteed flatten in a `finally` block. +- `tests/_diag_push.py` — captures real Push message bodies (quotes, portfolio, + ping, close-subscription). + +## 6. Known limitations / next steps + +- **Ping interval observation**: the server pinged ~2× over a ~1 min session; + a dedicated long-run ping test would confirm the cadence. +- **Explicit subscription close** is verified at the protocol level (unit test + + diag capture) but there is no SDK helper yet — `send_message()` is used with + the raw `*CloseSubscriptionRequest` payload. A `close_subscription()` helper + is a candidate addition. +- **Order lifecycle (Phase E of `TEST_PLAN_VELOTRADE.md`) is not yet tested** — + that requires a demo/sandbox account. The plan's E1–E12 cases are the next step. +- **`get_session_health()`** returns `last_activity` as a `datetime` (not + JSON-serializable); harmless, but noted. +- Offline quality gates: `ruff` clean on new files; `black` clean on new files; + `mypy` still red tree-wide (pre-existing 272 errors; the new code's errors + were fixed — count dropped to 90 for the two touched modules). diff --git a/tests/_diag_push.py b/tests/_diag_push.py new file mode 100644 index 0000000..bd79343 --- /dev/null +++ b/tests/_diag_push.py @@ -0,0 +1,198 @@ +"""Diagnostic: capture real Velotrade Push message bodies for tests.md. + +Run: PYTHONPATH=src venv/Scripts/python.exe tests/_diag_push.py +""" + +import asyncio +import json +import sys + +sys.path.insert(0, "src") + +from dxtrade import create_transport + +MD_URL = "wss://dx.velotrade.com/dxsca-web/md?format=JSON" +PF_URL = "wss://dx.velotrade.com/dxsca-web/?format=JSON" + + +def extract_account_code(users): + def find_in(item): + if isinstance(item, dict): + for key in ("accountCode", "account", "id"): + v = item.get(key) + if isinstance(v, str) and v.startswith("default:"): + return v + for key in ("accounts", "users", "userDetails", "accountList"): + v = item.get(key) + if isinstance(v, list): + for e in v: + f = find_in(e) + if f: + return f + elif isinstance(item, list): + for e in item: + f = find_in(e) + if f: + return f + return "" + + code = find_in(users) + if not code: + raise ValueError("no account code") + return code + + +def extract_symbols(instruments): + if isinstance(instruments, dict): + for key in ("instrumentDetails", "instruments", "symbols"): + v = instruments.get(key) + if ( + isinstance(v, list) + and v + and isinstance(v[0], dict) + and "symbol" in v[0] + ): + return [i["symbol"] for i in v] + for _key, v in instruments.items(): + if ( + isinstance(v, list) + and v + and isinstance(v[0], dict) + and "symbol" in v[0] + ): + return [i["symbol"] for i in v] + return [] + + +async def main(): + transport = create_transport() + try: + token = await transport.authenticate() + print("1. LOGIN -> sessionToken:", token[:12], "...") + + users = await transport.get_users() + account = extract_account_code(users) + print("2. /users -> account code:", account) + + instruments = await transport.query_instruments(account=account, limit=100) + symbols = extract_symbols(instruments) + print("3. instruments -> symbols:", symbols[:8], "...") + assert symbols + + # ---- Portfolio: legacy payload then spec payload ---- + pf_msgs = [] + pf_ready = asyncio.Event() + + def pf_cb(msg): + pf_msgs.append(msg) + pf_ready.set() + + await transport.subscribe("portfolio", pf_cb, ws_url=PF_URL) + for _ in range(180): + if "portfolio" in transport._websockets: + break + await asyncio.sleep(0.25) + + legacy = { + "type": "AccountPortfoliosSubscriptionRequest", + "requestId": "diag-legacy", + "session": token, + "payload": { + "account": account, + "eventTypes": [{"type": "Position", "format": "COMPACT"}], + }, + } + print("4. sending LEGACY portfolio payload:", json.dumps(legacy["payload"])) + await transport.send_message("portfolio", legacy) + await asyncio.sleep(6) + legacy_reply = next( + ( + m + for m in pf_msgs + if isinstance(m, dict) and m.get("inReplyTo") == "diag-legacy" + ), + None, + ) + print( + " legacy reply:", json.dumps(legacy_reply) if legacy_reply else "(none)" + ) + + pf_msgs.clear() + await transport.send_portfolio_subscription(account) + await asyncio.sleep(6) + snap = next( + ( + m + for m in pf_msgs + if isinstance(m, dict) and m.get("type") == "AccountPortfolios" + ), + None, + ) + print("5. spec portfolio reply:", json.dumps(snap)[:400] if snap else "(none)") + print( + " portfolio count:", + len(snap.get("payload", {}).get("portfolios", [])) if snap else "n/a", + ) + + # ---- Market data ---- + md_msgs = [] + md_ready = asyncio.Event() + + def md_cb(msg): + md_msgs.append(msg) + md_ready.set() + + await transport.subscribe("quotes", md_cb, ws_url=MD_URL) + for _ in range(180): + if "quotes" in transport._websockets: + break + await asyncio.sleep(0.25) + + await transport.send_market_data_subscription(symbols[:5], account) + try: + await asyncio.wait_for(md_ready.wait(), timeout=30) + except asyncio.TimeoutError: + print("6. no MarketData within 30s") + return + market = next( + m for m in md_msgs if isinstance(m, dict) and m.get("type") == "MarketData" + ) + print("6. MarketData sample:", json.dumps(market)[:500]) + + # ---- Ping stats ---- + print("7. ping stats (quotes):", json.dumps(transport.get_ping_stats("quotes"))) + print( + " session health:", + json.dumps(transport.get_session_health(), default=str), + ) + + # ---- Close subscription explicitly (spec close request) ---- + close_req = { + "type": "AccountPortfoliosCloseSubscriptionRequest", + "requestId": "diag-close", + "refRequestId": snap.get("inReplyTo") if snap else "unknown", + "session": token, + "timestamp": "2026-08-14T00:00:00.000Z", + } + pf_msgs.clear() + await transport.send_message("portfolio", close_req) + await asyncio.sleep(3) + close_reply = next( + ( + m + for m in pf_msgs + if isinstance(m, dict) and m.get("inReplyTo") == "diag-close" + ), + None, + ) + print( + "8. close subscription reply:", + json.dumps(close_reply)[:300] if close_reply else "(none)", + ) + finally: + await transport.close() + print("9. transport closed cleanly") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/_diag_trade.py b/tests/_diag_trade.py new file mode 100644 index 0000000..4c95623 --- /dev/null +++ b/tests/_diag_trade.py @@ -0,0 +1,317 @@ +"""Diagnostic: place a small BTCUSD buy with a $10 stop loss, then close it. + +Usage: + # Discovery + size calculation only (no order placed) + PYTHONPATH=src venv/Scripts/python.exe tests/_diag_trade.py --dry-run + + # Place a buy + protective stop, wait HOLD_SECONDS, close, verify flat + PYTHONPATH=src venv/Scripts/python.exe tests/_diag_trade.py + +This trades real money on the configured account. Not a pytest test. +The position is always closed (and working orders cancelled) in a finally +block, so a failure mid-way cannot leave the account open. +""" + +import asyncio +import json +import sys +import uuid + +MAX_LOSS_USD = 10.0 +SYMBOL = "BTCUSD" +HOLD_SECONDS = 30 + + +def extract_account_code(users): + def find_in(item): + if isinstance(item, dict): + for key in ("accountCode", "account", "id"): + v = item.get(key) + if isinstance(v, str) and v.startswith("default:"): + return v + for key in ("accounts", "users", "userDetails", "accountList"): + v = item.get(key) + if isinstance(v, list): + for e in v: + f = find_in(e) + if f: + return f + elif isinstance(item, list): + for e in item: + f = find_in(e) + if f: + return f + return "" + + code = find_in(users) + if not code: + raise ValueError("no account code in /users") + return code + + +def find_instrument(instruments): + """Locate the instrument record from an instruments query.""" + items = [] + if isinstance(instruments, dict): + for key in ("instrumentDetails", "instruments", "symbols"): + v = instruments.get(key) + if isinstance(v, list): + items = v + break + if not items: + for _key, v in instruments.items(): + if isinstance(v, list) and v and isinstance(v[0], dict): + items = v + break + elif isinstance(instruments, list): + items = instruments + + for item in items: + if isinstance(item, dict) and item.get("symbol") == SYMBOL: + return item + raise ValueError(f"no {SYMBOL} instrument in query response") + + +def extract_quote_price(market_data): + """Extract bid/ask from a POST /marketdata response.""" + events = market_data.get("events", []) if isinstance(market_data, dict) else [] + for ev in events: + if isinstance(ev, dict) and ev.get("type") == "Quote": + return ev.get("bid"), ev.get("ask") + return None, None + + +def as_list(payload, key): + """Normalise a positions/orders/metrics response into a list of dicts.""" + if isinstance(payload, list): + return [p for p in payload if isinstance(p, dict)] + if isinstance(payload, dict): + items = payload.get(key, []) + if isinstance(items, list): + return [p for p in items if isinstance(p, dict)] + return [] + + +def order_code(): + return f"vt-{uuid.uuid4().hex[:12]}" + + +async def flatten(transport, account): + """Close all positions and cancel working orders for the symbol.""" + positions = as_list(await transport.get_account_positions(account), "positions") + for p in positions: + if p.get("symbol") != SYMBOL: + continue + close = { + "orderCode": order_code(), + "type": "MARKET", + "instrument": SYMBOL, + "positionEffect": "CLOSE", + "positionCode": p["positionCode"], + "side": "SELL", + "tif": "GTC", + } + try: + await transport.place_order(account, close) + print(f" [cleanup] closed position {p['positionCode']}") + except Exception as exc: # noqa: BLE001 + print(f" [cleanup] close FAILED {p['positionCode']}: {exc!r}") + + await asyncio.sleep(1) + orders = as_list(await transport.get_account_orders(account), "orders") + for o in orders: + status = o.get("status") + if status not in ("COMPLETED", "CANCELED", "EXPIRED", "REJECTED"): + ref = o.get("orderCode") or o.get("orderId") + try: + await transport.cancel_order(account, str(ref)) + print(f" [cleanup] cancelled order {ref}") + except Exception as exc: # noqa: BLE001 + print(f" [cleanup] cancel FAILED {ref}: {exc!r}") + + +async def main(): + dry_run = "--dry-run" in sys.argv + from dxtrade import create_transport # noqa: E402 + + transport = create_transport() + position_code = None + stop_code = None + try: + token = await transport.authenticate() + print(f"[1] login ok, token {token[:12]}...") + + users = await transport.get_users() + account = extract_account_code(users) + print(f"[2] account: {account}") + + instruments = await transport.query_instruments(symbols=[SYMBOL], account=account) + inst = find_instrument(instruments) + print(f"[3] instrument: {json.dumps(inst, default=str)[:400]}") + + market_data = await transport.get_market_data([SYMBOL], account=account) + bid, ask = extract_quote_price(market_data) + print(f"[4] quote {SYMBOL}: bid={bid} ask={ask}") + if not bid: + raise RuntimeError("no quote received") + + qty = inst.get("minOrderSize") + notional = qty * bid + stop_distance = MAX_LOSS_USD / qty + print( + f"[5] size: qty={qty} (min={inst.get('minOrderSize')}) " + f"notional~${notional:.2f}" + ) + print( + f"[5b] stop distance for ${MAX_LOSS_USD} loss on {qty} {SYMBOL}: " + f"{stop_distance:,.2f} pts" + ) + + open_order = { + "orderCode": order_code(), + "type": "MARKET", + "instrument": SYMBOL, + "quantity": qty, + "positionEffect": "OPEN", + "side": "BUY", + "tif": "GTC", + } + print(f"[6] open order: {json.dumps(open_order)}") + + if dry_run: + print("[7] DRY RUN - no order placed") + return + + # --- Place the buy --- + placed = await transport.place_order(account, open_order) + print(f"[7] open order reply: {json.dumps(placed, default=str)[:200]}") + + # --- Confirm fill via positions --- + position = None + for _ in range(60): + positions = as_list(await transport.get_account_positions(account), "positions") + position = next((p for p in positions if p.get("symbol") == SYMBOL), None) + if position: + break + await asyncio.sleep(0.5) + if not position: + raise RuntimeError("BUY did not fill: no position appeared within 30s") + position_code = position["positionCode"] + + entry = position.get("openPrice") or position.get("averagePrice") + print( + f"[8] position open: code={position_code} " + f"qty={position.get('quantity')} side={position.get('side')} " + f"entry={entry}" + ) + + # --- Protective stop loss: STOP SELL at entry - stop_distance --- + # NOTE: closing STOP/LIMIT orders must NOT carry a quantity + # (errorCode 33 otherwise). + stop_price = round(float(entry) - stop_distance, 2) + stop_code = order_code() + stop_order = { + "orderCode": stop_code, + "type": "STOP", + "instrument": SYMBOL, + "positionEffect": "CLOSE", + "positionCode": position_code, + "side": "SELL", + "stopPrice": stop_price, + "tif": "GTC", + } + print(f"[9] stop order: {json.dumps(stop_order)}") + stop_reply = await transport.place_order(account, stop_order) + print(f"[9] stop order reply: {json.dumps(stop_reply, default=str)[:200]}") + + # --- Confirm the stop is working --- + # The orders list does not reliably echo the stop's orderCode, so also + # accept metrics.openOrdersCount >= 1 (any working order must be the + # stop, since the market BUY fills immediately). + stop_confirmed = False + for _ in range(30): + orders = as_list(await transport.get_account_orders(account), "orders") + stop_working = next( + (o for o in orders if o.get("orderCode") == stop_code), None + ) + if stop_working: + stop_confirmed = True + break + metrics = as_list(await transport.get_account_metrics(account), "metrics") + if metrics and metrics[0].get("openOrdersCount", 0) >= 1: + stop_confirmed = True + break + await asyncio.sleep(0.5) + print(f"[10] stop confirmed: {stop_confirmed}") + if not stop_confirmed: + raise RuntimeError("STOP loss order not confirmed as working") + + # --- Hold --- + print(f"[11] holding {HOLD_SECONDS}s...") + await asyncio.sleep(HOLD_SECONDS) + + # --- Close the position (market SELL CLOSE, full close) --- + close_order = { + "orderCode": order_code(), + "type": "MARKET", + "instrument": SYMBOL, + "positionEffect": "CLOSE", + "positionCode": position_code, + "side": "SELL", + "tif": "GTC", + } + print(f"[12] close order: {json.dumps(close_order)}") + closed = await transport.place_order(account, close_order) + print(f"[12] close order reply: {json.dumps(closed, default=str)[:200]}") + + # --- Cancel the protective stop (no longer needed) --- + # Expected to 400: protective stops auto-cancel when the position closes. + await asyncio.sleep(1) + try: + cancel_reply = await transport.cancel_order(account, stop_code) + print(f"[13] cancel stop reply: {json.dumps(cancel_reply, default=str)[:200]}") + except Exception as exc: # noqa: BLE001 + detail = str(exc) + status = "" + if "status=400" in detail: + status = "400 (already auto-cancelled)" + elif "status=" in detail: + status = detail.split("status=")[1].split(",")[0] + print(f"[13] cancel stop: {status or 'error'}") + + # --- Verify flat --- + await asyncio.sleep(2) + positions = as_list(await transport.get_account_positions(account), "positions") + open_positions = [p for p in positions if p.get("symbol") == SYMBOL] + orders = as_list(await transport.get_account_orders(account), "orders") + working = [ + o + for o in orders + if o.get("instrument") == SYMBOL + and o.get("status") not in ("COMPLETED", "CANCELED", "EXPIRED", "REJECTED") + ] + print( + f"[14] after close: open positions={len(open_positions)}, " + f"working orders={len(working)}" + ) + + metrics = as_list(await transport.get_account_metrics(account), "metrics") + if metrics: + m = metrics[0] + print( + f"[15] metrics: equity={m.get('equity')} balance={m.get('balance')} " + f"openPL={m.get('openPL')} totalPL={m.get('totalPL')}" + ) + finally: + if not dry_run: + print("[16] cleanup (flatten + cancel working orders)...") + try: + await flatten(transport, account) + except Exception as exc: # noqa: BLE001 + print(f"[16] cleanup error: {exc!r}") + await transport.close() + print("[17] transport closed") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/test_auth.py b/tests/test_auth.py index 5f34921..51612ee 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -52,6 +52,12 @@ async def test_authenticate_request(self, bearer_token_credentials): # Verify authentication header was added assert authenticated_request.headers["Authorization"] == "Bearer test_bearer_token" assert authenticated_request == request # Same request object should be returned + + def test_get_auth_headers(self, bearer_token_credentials): + """Test transport-agnostic auth headers.""" + handler = BearerTokenHandler(bearer_token_credentials) + headers = handler.get_auth_headers() + assert headers == {"Authorization": "Bearer test_bearer_token"} class TestHMACHandler: @@ -169,7 +175,45 @@ async def test_authenticate_request_without_passphrase(self): assert authenticated_request.headers["DX-API-SIGNATURE"] == expected_signature_b64 -class TestSessionHandler: +class TestHMACGetAuthHeaders: + """Test transport-agnostic HMAC auth headers.""" + + def test_get_auth_headers_with_passphrase(self, hmac_credentials): + """Test auth headers include signature and passphrase.""" + handler = HMACHandler(hmac_credentials) + headers = handler.get_auth_headers(method="GET", path="/api/accounts", body="") + + assert headers["DX-API-KEY"] == "test_api_key" + assert headers["DX-API-PASSPHRASE"] == "test_passphrase" + + timestamp = headers["DX-API-TIMESTAMP"] + expected_signature_string = ( + f"{timestamp}GET/api/accountstest_passphrase" + ) + expected_signature = hmac.new( + "test_secret_key".encode("utf-8"), + expected_signature_string.encode("utf-8"), + hashlib.sha256, + ).digest() + assert headers["DX-API-SIGNATURE"] == b64encode(expected_signature).decode("utf-8") + + def test_get_auth_headers_without_passphrase(self): + """Test auth headers without passphrase.""" + credentials = HMACCredentials( + api_key="test_api_key", + secret_key="test_secret_key", + ) + handler = HMACHandler(credentials) + headers = handler.get_auth_headers(method="GET", path="/api/accounts", body="") + + assert "DX-API-PASSPHRASE" not in headers + timestamp = headers["DX-API-TIMESTAMP"] + expected_signature = hmac.new( + "test_secret_key".encode("utf-8"), + f"{timestamp}GET/api/accounts".encode("utf-8"), + hashlib.sha256, + ).digest() + assert headers["DX-API-SIGNATURE"] == b64encode(expected_signature).decode("utf-8") """Test session authentication handler.""" def test_init_with_valid_credentials(self, session_credentials): @@ -210,6 +254,23 @@ async def test_authenticate_with_valid_token(self, session_credentials): assert authenticated_request.headers["X-Auth-Token"] == "valid_token" assert authenticated_request.headers["Authorization"] == "DXAPI valid_token" + + def test_get_auth_headers_with_token(self, session_credentials): + """Test auth headers with an active token.""" + handler = SessionHandler(session_credentials) + handler._session_token = "valid_token" + handler._token_expires_at = time.time() + 3600 + handler._last_login = time.time() + + headers = handler.get_auth_headers() + + assert headers["X-Auth-Token"] == "valid_token" + assert headers["Authorization"] == "DXAPI valid_token" + + def test_get_auth_headers_without_token(self, session_credentials): + """Test auth headers before any token is available.""" + handler = SessionHandler(session_credentials) + assert handler.get_auth_headers() == {} @pytest.mark.asyncio async def test_authenticate_requires_login(self, session_credentials): diff --git a/tests/test_transport.py b/tests/test_transport.py new file mode 100644 index 0000000..4007ffc --- /dev/null +++ b/tests/test_transport.py @@ -0,0 +1,466 @@ +"""Unit tests for the transport layer's DXTrade REST API methods. + +The aiohttp session is mocked — these tests never touch the network. +""" + +import json +import time +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +import pytest + +from dxtrade.config import AuthConfig +from dxtrade.config import AuthType +from dxtrade.config import SDKConfig +from dxtrade.transport import DXTradeTransport + +BASE_URL = "https://broker.example/dxsca-web" +SESSION_TOKEN = "test-session-token" + + +def _mock_response(status=200, payload=None, content_type="application/json"): + """Build an aiohttp-style response mock usable in an async context.""" + response = AsyncMock() + response.status = status + response.headers = {"content-type": content_type} + response.raise_for_status = MagicMock() + if payload is not None: + response.json = AsyncMock(return_value=payload) + response.__aenter__ = AsyncMock(return_value=response) + response.__aexit__ = AsyncMock(return_value=False) + return response + + +@pytest.fixture +def transport(): + """Transport with a real config, an active token and a mocked session.""" + config = SDKConfig( + base_url=BASE_URL, + auth=AuthConfig( + type=AuthType.CREDENTIALS, + username="test_user", + password="test_password", + domain="default", + ), + ) + transport_ = DXTradeTransport(config) + transport_.auth_handler._session_token = SESSION_TOKEN + transport_.auth_handler._token_expires_at = time.time() + 3600 + transport_.auth_handler._last_login = time.time() + # MagicMock (not AsyncMock): the transport uses + # "async with session.request(...)", which needs a plain call returning + # an async context manager rather than a coroutine. + transport_._session = MagicMock() + return transport_ + + +def _expected_headers(): + return { + "X-Auth-Token": SESSION_TOKEN, + "Authorization": f"DXAPI {SESSION_TOKEN}", + } + + +class TestEncodeAccount: + """Test account code URL encoding.""" + + def test_encodes_colon(self): + """The colon in an account code must be percent-encoded.""" + assert DXTradeTransport._encode_account("default:12345") == "default%3A12345" + + def test_plain_account_unchanged(self): + """Account codes without special characters are unchanged.""" + assert DXTradeTransport._encode_account("abc") == "abc" + + +class TestUsers: + """Test account discovery.""" + + async def test_get_users(self, transport): + """GET /users with the session auth headers.""" + payload = {"users": [{"accountCode": "default:12345"}]} + transport._session.request.return_value = _mock_response(payload=payload) + + result = await transport.get_users() + + assert result == payload + transport._session.request.assert_called_once_with( + "GET", + f"{BASE_URL}/users", + headers=_expected_headers(), + ) + + +class TestAccountScoped: + """Test account-scoped REST resources.""" + + async def test_get_account_metrics(self, transport): + """GET /accounts/{encoded}/metrics.""" + transport._session.request.return_value = _mock_response( + payload={"account": "default:12345"} + ) + + result = await transport.get_account_metrics("default:12345") + + assert result == {"account": "default:12345"} + transport._session.request.assert_called_once_with( + "GET", + f"{BASE_URL}/accounts/default%3A12345/metrics", + headers=_expected_headers(), + ) + + async def test_get_account_portfolio(self, transport): + """GET /accounts/{encoded}/portfolio.""" + transport._session.request.return_value = _mock_response( + payload={"portfolios": []} + ) + + result = await transport.get_account_portfolio("default:12345") + + assert result == {"portfolios": []} + transport._session.request.assert_called_once_with( + "GET", + f"{BASE_URL}/accounts/default%3A12345/portfolio", + headers=_expected_headers(), + ) + + async def test_get_account_positions(self, transport): + """GET /accounts/{encoded}/positions.""" + transport._session.request.return_value = _mock_response(payload=[]) + + result = await transport.get_account_positions("default:12345") + + assert result == [] + transport._session.request.assert_called_once_with( + "GET", + f"{BASE_URL}/accounts/default%3A12345/positions", + headers=_expected_headers(), + ) + + async def test_get_account_orders(self, transport): + """GET /accounts/{encoded}/orders.""" + transport._session.request.return_value = _mock_response(payload=[]) + + result = await transport.get_account_orders("default:12345") + + assert result == [] + transport._session.request.assert_called_once_with( + "GET", + f"{BASE_URL}/accounts/default%3A12345/orders", + headers=_expected_headers(), + ) + + async def test_get_account_orders_history(self, transport): + """GET /accounts/{encoded}/orders/history.""" + transport._session.request.return_value = _mock_response(payload=[]) + + result = await transport.get_account_orders_history("default:12345") + + assert result == [] + transport._session.request.assert_called_once_with( + "GET", + f"{BASE_URL}/accounts/default%3A12345/orders/history", + headers=_expected_headers(), + ) + + +class TestInstruments: + """Test instrument discovery.""" + + async def test_query_instruments_account_scoped(self, transport): + """Account-scoped query passes symbols/limit as query params.""" + transport._session.request.return_value = _mock_response( + payload={"instrumentDetails": []} + ) + + result = await transport.query_instruments( + symbols=["BTC", "ETH"], account="default:12345", limit=5 + ) + + assert result == {"instrumentDetails": []} + transport._session.request.assert_called_once_with( + "GET", + f"{BASE_URL}/accounts/default%3A12345/instruments/query", + headers=_expected_headers(), + params={"symbols": "BTC,ETH", "limit": 5}, + ) + + async def test_query_instruments_global(self, transport): + """Without an account, the global instruments query is used.""" + transport._session.request.return_value = _mock_response(payload={}) + + result = await transport.query_instruments(symbols=["BTC"]) + + assert result == {} + transport._session.request.assert_called_once_with( + "GET", + f"{BASE_URL}/instruments/query", + headers=_expected_headers(), + params={"symbols": "BTC"}, + ) + + async def test_query_instruments_no_args(self, transport): + """No arguments still produces a valid request.""" + transport._session.request.return_value = _mock_response(payload={}) + + result = await transport.query_instruments() + + assert result == {} + transport._session.request.assert_called_once_with( + "GET", + f"{BASE_URL}/instruments/query", + headers=_expected_headers(), + params={}, + ) + + +class TestMarketData: + """Test REST market data snapshot.""" + + async def test_get_market_data_defaults(self, transport): + """POST /marketdata with default Quote/COMPACT event types.""" + transport._session.request.return_value = _mock_response(payload={"events": []}) + + result = await transport.get_market_data(["EUR/USD"]) + + assert result == {"events": []} + transport._session.request.assert_called_once_with( + "POST", + f"{BASE_URL}/marketdata", + headers=_expected_headers(), + json={ + "symbols": ["EUR/USD"], + "eventTypes": [{"type": "Quote", "format": "COMPACT"}], + }, + ) + + async def test_get_market_data_with_account_and_types(self, transport): + """Account and custom event types are forwarded.""" + transport._session.request.return_value = _mock_response(payload={}) + + await transport.get_market_data( + ["EUR/USD"], + event_types=[{"type": "Candle", "candleType": "5m", "format": "COMPACT"}], + account="default:12345", + ) + + transport._session.request.assert_called_once_with( + "POST", + f"{BASE_URL}/marketdata", + headers=_expected_headers(), + json={ + "symbols": ["EUR/USD"], + "eventTypes": [ + {"type": "Candle", "candleType": "5m", "format": "COMPACT"} + ], + "account": "default:12345", + }, + ) + + +class TestPing: + """Test session validation and token refresh.""" + + async def test_ping_refreshes_token(self, transport): + """A new sessionToken from /ping updates the stored token.""" + transport._session.request.return_value = _mock_response( + payload={"sessionToken": "fresh-token"} + ) + + result = await transport.ping() + + assert result == {"sessionToken": "fresh-token"} + assert transport.auth_handler.get_session_token() == "fresh-token" + + async def test_ping_without_token(self, transport): + """A ping response without sessionToken leaves the token untouched.""" + transport._session.request.return_value = _mock_response(payload={}) + + await transport.ping() + + assert transport.auth_handler.get_session_token() == SESSION_TOKEN + + +class TestLogout: + """Test session invalidation.""" + + async def test_logout_clears_token(self, transport): + """POST /logout and clear the local token.""" + transport._session.request.return_value = _mock_response(payload={}) + + await transport.logout() + + transport._session.request.assert_called_once_with( + "POST", + f"{BASE_URL}/logout", + headers=_expected_headers(), + ) + assert transport.auth_handler.get_session_token() is None + + +class TestSubscriptionMessages: + """Test the Push subscription message builders.""" + + async def test_market_data_subscription_shape(self, transport): + """Market data subscription carries requestId, timestamp, session and payload.""" + websocket = AsyncMock() + transport._websockets["quotes"] = websocket + + await transport.send_market_data_subscription(["EUR/USD"], "default:12345") + + sent = json.loads(websocket.send.call_args[0][0]) + assert sent["type"] == "MarketDataSubscriptionRequest" + assert sent["requestId"] + assert sent["timestamp"] + assert sent["session"] == SESSION_TOKEN + assert sent["payload"] == { + "account": "default:12345", + "symbols": ["EUR/USD"], + "eventTypes": [{"type": "Quote", "format": "COMPACT"}], + } + + async def test_portfolio_subscription_shape(self, transport): + """Portfolio subscription uses the DXTrade spec payload (requestType/accounts).""" + websocket = AsyncMock() + transport._websockets["portfolio"] = websocket + + await transport.send_portfolio_subscription("default:12345") + + sent = json.loads(websocket.send.call_args[0][0]) + assert sent["type"] == "AccountPortfoliosSubscriptionRequest" + assert sent["requestId"] + assert sent["timestamp"] + assert sent["session"] == SESSION_TOKEN + assert sent["payload"] == { + "requestType": "LIST", + "accounts": ["default:12345"], + } + + +class TestSendMessage: + """Test raw message sending without blocking on a reply.""" + + async def test_send_message_sends_without_recv(self, transport): + """send_message only sends; the handler loop owns recv.""" + websocket = AsyncMock() + transport._websockets["quotes"] = websocket + + result = await transport.send_message( + "quotes", {"type": "MarketDataSubscriptionRequest", "requestId": "abc"} + ) + + assert result is None + websocket.send.assert_called_once_with( + '{"type": "MarketDataSubscriptionRequest", "requestId": "abc"}' + ) + websocket.recv.assert_not_called() + + async def test_send_message_requires_connected_channel(self, transport): + """Sending to an unknown channel raises ValueError.""" + with pytest.raises(ValueError): + await transport.send_message("nope", {"type": "x"}) + + async def test_send_message_accepts_string(self, transport): + """Raw string messages pass through unchanged.""" + websocket = AsyncMock() + transport._websockets["quotes"] = websocket + + await transport.send_message("quotes", '{"type": "ping"}') + + websocket.send.assert_called_once_with('{"type": "ping"}') + + +class TestPingPongHandler: + """Test the application-level ping/pong handler.""" + + async def test_handles_ping_request(self, transport): + """A PingRequest is answered with a Ping and not forwarded.""" + websocket = AsyncMock() + forwarded = [] + transport._subscriptions["quotes"] = forwarded.append + + handled = await transport._handle_ping_pong( + "quotes", + {"type": "PingRequest", "session": SESSION_TOKEN, "timestamp": "t"}, + websocket, + SESSION_TOKEN, + ) + + assert handled is True + assert forwarded == [], "ping messages must not reach the user callback" + sent = json.loads(websocket.send.call_args[0][0]) + assert sent["type"] == "Ping" + assert sent["session"] == SESSION_TOKEN + assert sent["timestamp"] + + stats = transport.get_ping_stats("quotes") + assert stats["ping_requests_received"] == 1 + assert stats["ping_responses_sent"] == 1 + assert stats["session_extensions"] == 1 + + async def test_ignores_regular_message(self, transport): + """Regular messages are not treated as pings.""" + websocket = AsyncMock() + + handled = await transport._handle_ping_pong( + "quotes", {"type": "MarketData", "payload": {}}, websocket, SESSION_TOKEN + ) + + assert handled is False + websocket.send.assert_not_called() + + +class TestPlaceOrder: + """Test order placement.""" + + async def test_place_order_open(self, transport): + """POST /accounts/{encoded}/orders with the order payload.""" + transport._session.request.return_value = _mock_response( + payload={"orderId": 1, "status": "ACCEPTED"} + ) + order = { + "orderCode": "vt-abc123", + "type": "MARKET", + "instrument": "BTCUSDT", + "quantity": 0.0001, + "positionEffect": "OPEN", + "side": "BUY", + "tif": "GTC", + } + + result = await transport.place_order("default:12345", order) + + assert result == {"orderId": 1, "status": "ACCEPTED"} + expected_payload = dict(order) + expected_payload["account"] = "default:12345" + transport._session.request.assert_called_once_with( + "POST", + f"{BASE_URL}/accounts/default%3A12345/orders", + headers=_expected_headers(), + json=expected_payload, + ) + + async def test_place_order_close_position(self, transport): + """A closing order carries positionEffect CLOSE and positionCode.""" + transport._session.request.return_value = _mock_response(payload={}) + close_order = { + "orderCode": "vt-close1", + "type": "MARKET", + "instrument": "BTCUSDT", + "positionEffect": "CLOSE", + "positionCode": "pos-1", + "side": "SELL", + "tif": "GTC", + } + + await transport.place_order("default:12345", close_order) + + expected_payload = dict(close_order) + expected_payload["account"] = "default:12345" + transport._session.request.assert_called_once_with( + "POST", + f"{BASE_URL}/accounts/default%3A12345/orders", + headers=_expected_headers(), + json=expected_payload, + ) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..74f31c1 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,267 @@ +"""Unit tests for the high-level transport utilities.""" + +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +import pytest + +from dxtrade.utils import account_is_flat +from dxtrade.utils import close_position +from dxtrade.utils import find_account_code +from dxtrade.utils import flatten +from dxtrade.utils import open_position +from dxtrade.utils import resolve_account +from dxtrade.utils import resolve_symbol +from dxtrade.utils import stream_quotes + + +@pytest.fixture +def transport(): + """AsyncMock transport with a valid session and one account.""" + mock = AsyncMock() + mock.auth_handler = MagicMock() + mock.auth_handler.get_session_token.return_value = "token" + mock.get_users.return_value = {"users": [{"accountCode": "default:12345"}]} + return mock + + +class TestFindAccountCode: + """Test account code extraction from /users responses.""" + + def test_list_shape(self): + users = [{"accountCode": "default:12345"}] + assert find_account_code(users) == "default:12345" + + def test_nested_accounts(self): + users = {"users": [{"accounts": [{"accountCode": "default:12345"}]}]} + assert find_account_code(users) == "default:12345" + + def test_no_account(self): + with pytest.raises(ValueError): + find_account_code({"users": []}) + + +class TestResolveAccount: + """Test account discovery through the transport.""" + + async def test_uses_existing_token(self, transport): + account = await resolve_account(transport) + assert account == "default:12345" + transport.authenticate.assert_not_called() + + async def test_logs_in_when_no_token(self, transport): + transport.auth_handler.get_session_token.return_value = None + account = await resolve_account(transport) + assert account == "default:12345" + transport.authenticate.assert_awaited_once() + + +class TestResolveSymbol: + """Test symbol hint resolution.""" + + async def test_exact_match(self, transport): + transport.query_instruments.return_value = { + "instrumentDetails": [{"symbol": "BTCUSD", "minOrderSize": 0.001}] + } + symbol = await resolve_symbol(transport, "default:12345", "BTCUSD") + assert symbol == "BTCUSD" + + async def test_usdt_hint_matches_usd_symbol(self, transport): + # The server matches exactly: BTCUSDT returns nothing, BTCUSD returns + # the instrument record. + def query_side_effect(symbols=None, account=None, limit=None): + if symbols == ["BTCUSD"]: + return { + "instrumentDetails": [{"symbol": "BTCUSD", "minOrderSize": 0.001}] + } + return {"instrumentDetails": []} + + transport.query_instruments.side_effect = query_side_effect + symbol = await resolve_symbol(transport, "default:12345", "BTCUSDT") + assert symbol == "BTCUSD" + # BTCUSDT (no match) and BTCUSD (match) were both queried + queried = [ + call.kwargs["symbols"] + for call in transport.query_instruments.call_args_list + ] + assert ["BTCUSDT"] in queried + assert ["BTCUSD"] in queried + + async def test_unresolvable_raises(self, transport): + transport.query_instruments.return_value = {"instrumentDetails": []} + with pytest.raises(ValueError): + await resolve_symbol(transport, "default:12345", "NOPE") + + +class TestOpenPosition: + """Test opening a position with optional protective stop.""" + + def _transport(self, transport): + transport.query_instruments.return_value = { + "instrumentDetails": [{"symbol": "BTCUSD", "minOrderSize": 0.001}] + } + transport.get_market_data.return_value = { + "events": [{"type": "Quote", "bid": 60000.0, "ask": 60001.0}] + } + transport.get_account_positions.return_value = { + "positions": [ + { + "positionCode": "P1", + "symbol": "BTCUSD", + "quantity": 0.001, + "side": "BUY", + "openPrice": 60001.0, + } + ] + } + transport.place_order.return_value = {"orderId": 100} + return transport + + async def test_open_without_stop(self, transport): + self._transport(transport) + result = await open_position(transport, "BTCUSD") + + assert result["position"]["positionCode"] == "P1" + assert result["stop_order"] is None + assert transport.place_order.await_count == 1 + + async def test_open_with_usd_stop_loss(self, transport): + self._transport(transport) + result = await open_position(transport, "BTCUSD", stop_loss=10.0) + + assert transport.place_order.await_count == 2 + # stop price = entry (ask) - loss/qty = 60001 - 10/0.001 = 50001 + stop_payload = transport.place_order.await_args_list[1].args[1] + assert stop_payload["type"] == "STOP" + assert stop_payload["positionEffect"] == "CLOSE" + assert stop_payload["stopPrice"] == 50001.0 + assert stop_payload["side"] == "SELL" + # closing STOP orders must not carry a quantity + assert "quantity" not in stop_payload + assert result["stop_order"] == {"orderId": 100} + + async def test_open_with_absolute_stop_price(self, transport): + self._transport(transport) + await open_position(transport, "BTCUSD", stop_loss_price=55000.0) + + stop_payload = transport.place_order.await_args_list[1].args[1] + assert stop_payload["stopPrice"] == 55000.0 + + async def test_bad_side(self, transport): + with pytest.raises(ValueError): + await open_position(transport, "BTCUSD", side="SIDEWAYS") + + async def test_no_fill_raises(self, transport): + self._transport(transport) + transport.get_account_positions.return_value = {"positions": []} + with pytest.raises(RuntimeError): + await open_position(transport, "BTCUSD", fill_timeout=0.2) + + +class TestClosePosition: + """Test closing a position.""" + + async def test_close_by_symbol(self, transport): + transport.get_account_positions.return_value = { + "positions": [ + { + "positionCode": "P1", + "symbol": "BTCUSD", + "quantity": 0.001, + "side": "BUY", + } + ] + } + transport.place_order.return_value = {"orderId": 200} + + result = await close_position(transport, symbol="BTCUSD") + + payload = transport.place_order.await_args.args[1] + assert payload["positionEffect"] == "CLOSE" + assert payload["positionCode"] == "P1" + assert payload["side"] == "SELL" + assert "quantity" not in payload + assert result["position"]["positionCode"] == "P1" + + async def test_close_requires_exactly_one(self, transport): + transport.get_account_positions.return_value = {"positions": []} + with pytest.raises(ValueError): + await close_position(transport, symbol="BTCUSD") + + +class TestFlatten: + """Test flattening the account.""" + + async def test_closes_and_cancels(self, transport): + transport.get_account_positions.return_value = { + "positions": [ + {"positionCode": "P1", "symbol": "BTCUSD", "side": "BUY"}, + {"positionCode": "P2", "symbol": "ETHUSD", "side": "SELL"}, + ] + } + transport.get_account_orders.return_value = { + "orders": [ + {"orderCode": "o1", "status": "WORKING"}, + {"orderCode": "o2", "status": "COMPLETED"}, + ] + } + + result = await flatten(transport) + + assert result["closed"] == ["P1", "P2"] + assert result["cancelled"] == ["o1"] + assert not result["errors"] + + +class TestAccountIsFlat: + """Test the flat-state check.""" + + async def test_flat(self, transport): + transport.get_account_metrics.return_value = { + "metrics": [{"openPositionsCount": 0, "openOrdersCount": 0}] + } + assert await account_is_flat(transport) is True + + async def test_not_flat(self, transport): + transport.get_account_metrics.return_value = { + "metrics": [{"openPositionsCount": 1, "openOrdersCount": 0}] + } + assert await account_is_flat(transport) is False + + +class TestStreamQuotes: + """Test quote streaming.""" + + async def test_streams_and_collects(self, transport): + transport.query_instruments.return_value = { + "instrumentDetails": [{"symbol": "BTCUSD", "minOrderSize": 0.001}] + } + transport.wait_for_channel.return_value = True + + captured = {} + + async def fake_subscribe(channel, callback, **kwargs): + captured["callback"] = callback + + transport.subscribe.side_effect = fake_subscribe + + # A short real sleep is fine; no network involved. + events = await stream_quotes(transport, symbols=["BTCUSD"], duration=0.01) + + assert events == [] + # Deliver a quote through the captured callback + callback = captured["callback"] + callback( + { + "type": "MarketData", + "payload": { + "events": [ + {"symbol": "BTCUSD", "bid": 1.0, "ask": 1.1, "time": "t"} + ] + }, + } + ) + transport.send_market_data_subscription.assert_awaited_once_with( + ["BTCUSD"], "default:12345" + ) + transport.unsubscribe.assert_awaited_once_with("quotes") diff --git a/tests/test_velotrade_live.py b/tests/test_velotrade_live.py new file mode 100644 index 0000000..76aeba2 --- /dev/null +++ b/tests/test_velotrade_live.py @@ -0,0 +1,126 @@ +"""Live smoke tests: authenticate against a real broker via the SDK transport. + +These tests require a broker ``.env`` in the repository root with ``DXTRADE_*`` +variables set (e.g. Velotrade). They are skipped otherwise and never run in CI. +""" + +import os +from pathlib import Path +from typing import Any + +import pytest +from dotenv import load_dotenv + +from dxtrade import create_transport + +load_dotenv() + +pytestmark = pytest.mark.skipif( + not Path(".env").exists() or not os.getenv("DXTRADE_USERNAME"), + reason="requires a broker .env with DXTRADE_USERNAME", +) + + +def _extract_account_code(users: Any) -> str: + """Best-effort extraction of the first account code from a /users response. + + DXTrade account codes look like ``default:12345``. The exact /users + response shape varies between brokers, so several shapes are handled. + + Args: + users: Raw /users response (dict or list) + + Returns: + First account code found + + Raises: + ValueError: No account code could be located + """ + + def find_in(item: Any) -> str: + if isinstance(item, dict): + for key in ("accountCode", "account", "id"): + value = item.get(key) + if isinstance(value, str) and value.startswith("default:"): + return value + for key in ("accounts", "users", "userDetails", "accountList"): + value = item.get(key) + if isinstance(value, list): + for entry in value: + found = find_in(entry) + if found: + return found + elif isinstance(item, list): + for entry in item: + found = find_in(entry) + if found: + return found + return "" + + code = find_in(users) + if not code: + raise ValueError("Could not find an account code in /users response") + return code + + +class TestLiveLogin: + """Live login smoke tests against the configured broker.""" + + async def test_authenticate_returns_session_token(self): + """Login with SDK credentials and obtain a session token.""" + transport = create_transport() + try: + token = await transport.authenticate() + + assert token, "authenticate() returned an empty token" + assert isinstance(token, str) + assert len(token) >= 10, "session token suspiciously short" + finally: + await transport.close() + + async def test_authenticated_request_uses_token(self): + """After login, a REST request carries the session token.""" + transport = create_transport() + try: + token = await transport.authenticate() + assert token + + # /users is the account-discovery endpoint on DXTrade brokers. + result = await transport.get_users() + assert result is not None + finally: + await transport.close() + + async def test_full_flow_users_and_instruments(self): + """Mirror the reference direct-API flow entirely through the SDK. + + login -> /users (account discovery) -> account-scoped instruments. + """ + transport = create_transport() + try: + await transport.authenticate() + + users = await transport.get_users() + account_code = _extract_account_code(users) + assert account_code.startswith("default:") + + instruments = await transport.query_instruments( + account=account_code, limit=5 + ) + assert instruments is not None + finally: + await transport.close() + + async def test_ping_and_logout(self): + """Session validation via /ping and clean logout.""" + transport = create_transport() + try: + await transport.authenticate() + + ping_result = await transport.ping() + assert ping_result is not None + + await transport.logout() + assert transport.auth_handler.get_session_token() is None + finally: + await transport.close() diff --git a/tests/test_velotrade_ws_live.py b/tests/test_velotrade_ws_live.py new file mode 100644 index 0000000..43cac18 --- /dev/null +++ b/tests/test_velotrade_ws_live.py @@ -0,0 +1,275 @@ +"""Live WebSocket (Push API) tests against a real broker via the SDK transport. + +These tests require a broker ``.env`` in the repository root with ``DXTRADE_*`` +variables set (e.g. Velotrade). They are skipped otherwise and never run in CI. +""" + +import asyncio +import os +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest +from dotenv import load_dotenv + +from dxtrade import create_transport + +load_dotenv() + +pytestmark = pytest.mark.skipif( + not Path(".env").exists() or not os.getenv("DXTRADE_USERNAME"), + reason="requires a broker .env with DXTRADE_USERNAME", +) + +# Velotrade connection values (see TEST_PLAN_VELOTRADE.md §2) +MARKET_DATA_URL = "wss://dx.velotrade.com/dxsca-web/md?format=JSON" +PORTFOLIO_URL = "wss://dx.velotrade.com/dxsca-web/?format=JSON" + + +def _extract_account_code(users: Any) -> str: + """Best-effort extraction of the first account code from a /users response.""" + + def find_in(item: Any) -> str: + if isinstance(item, dict): + for key in ("accountCode", "account", "id"): + value = item.get(key) + if isinstance(value, str) and value.startswith("default:"): + return value + for key in ("accounts", "users", "userDetails", "accountList"): + value = item.get(key) + if isinstance(value, list): + for entry in value: + found = find_in(entry) + if found: + return found + elif isinstance(item, list): + for entry in item: + found = find_in(entry) + if found: + return found + return "" + + code = find_in(users) + if not code: + raise ValueError("Could not find an account code in /users response") + return code + + +def _extract_symbols(instruments: Any) -> list[str]: + """Best-effort extraction of instrument symbols from an instruments query.""" + if isinstance(instruments, dict): + for key in ("instrumentDetails", "instruments", "symbols"): + value = instruments.get(key) + if isinstance(value, list): + if all(isinstance(item, dict) and "symbol" in item for item in value): + return [item["symbol"] for item in value] + if all(isinstance(item, str) for item in value): + return value + for _key, value in instruments.items(): + if ( + isinstance(value, list) + and value + and isinstance(value[0], dict) + and "symbol" in value[0] + ): + return [item["symbol"] for item in value] + elif isinstance(instruments, list): + return [ + item["symbol"] + for item in instruments + if isinstance(item, dict) and "symbol" in item + ] + return [] + + +async def _wait_for_channel(transport, channel: str, timeout: float = 45.0): + """Wait until the transport has an open WebSocket for the channel.""" + loop = asyncio.get_event_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + if channel in transport._websockets: + return + await asyncio.sleep(0.25) + raise TimeoutError(f"WebSocket channel '{channel}' did not connect") + + +def _make_callback(ready: asyncio.Event, received: list[dict]) -> Callable: + """Build a message callback that records messages and flags a type.""" + + def callback(message: Any) -> None: + received.append(message) + if isinstance(message, dict): + ready.set() + + return callback + + +class TestMarketDataPush: + """Live market-data Push tests.""" + + async def test_market_data_subscription_streams_quotes(self): + """Subscribe to quotes and receive MarketData events for our symbols.""" + transport = create_transport() + try: + token = await transport.authenticate() + assert token + + users = await transport.get_users() + account = _extract_account_code(users) + + instruments = await transport.query_instruments(account=account, limit=100) + symbols = _extract_symbols(instruments)[:5] + assert symbols, "no symbols discovered from instruments query" + + received: list[dict] = [] + ready = asyncio.Event() + + await transport.subscribe( + "quotes", _make_callback(ready, received), ws_url=MARKET_DATA_URL + ) + await _wait_for_channel(transport, "quotes") + + await transport.send_market_data_subscription(symbols, account) + + await asyncio.wait_for(ready.wait(), timeout=60) + + market = next( + m + for m in received + if isinstance(m, dict) and m.get("type") == "MarketData" + ) + events = market.get("payload", {}).get("events", []) + assert events, "MarketData message carried no events" + + quoted = {e.get("symbol") for e in events if isinstance(e, dict)} + assert quoted & set( + symbols + ), f"no events for requested symbols {symbols}; got {quoted}" + assert any( + isinstance(e, dict) and ("bid" in e or "ask" in e) for e in events + ), "no bid/ask fields in quote events" + finally: + await transport.close() + + +class TestPortfolioPush: + """Live portfolio Push tests.""" + + async def test_portfolio_subscription_receives_snapshot(self): + """Spec-shaped portfolio subscription returns an AccountPortfolios snapshot.""" + transport = create_transport() + try: + await transport.authenticate() + users = await transport.get_users() + account = _extract_account_code(users) + + received: list[dict] = [] + ready = asyncio.Event() + + await transport.subscribe( + "portfolio", _make_callback(ready, received), ws_url=PORTFOLIO_URL + ) + await _wait_for_channel(transport, "portfolio") + + await transport.send_portfolio_subscription(account) + + await asyncio.wait_for(ready.wait(), timeout=60) + + portfolios = next( + m + for m in received + if isinstance(m, dict) and m.get("type") == "AccountPortfolios" + ) + payload = portfolios.get("payload", {}) + assert isinstance(payload.get("portfolios"), list) + finally: + await transport.close() + + async def test_portfolio_legacy_payload_rejected(self): + """The pre-fix SDK payload shape is rejected by the server (gap G5).""" + transport = create_transport() + try: + await transport.authenticate() + users = await transport.get_users() + account = _extract_account_code(users) + + received: list[dict] = [] + ready = asyncio.Event() + + await transport.subscribe( + "portfolio", _make_callback(ready, received), ws_url=PORTFOLIO_URL + ) + await _wait_for_channel(transport, "portfolio") + + # Old SDK shape: payload {account, eventTypes} — no requestType/accounts. + legacy = { + "type": "AccountPortfoliosSubscriptionRequest", + "requestId": "legacy-payload-test", + "session": transport.auth_handler.get_session_token(), + "payload": { + "account": account, + "eventTypes": [{"type": "Position", "format": "COMPACT"}], + }, + } + await transport.send_message("portfolio", legacy) + + try: + await asyncio.wait_for(ready.wait(), timeout=15) + except asyncio.TimeoutError: + pytest.fail( + "no response to legacy payload within 15s " + "(unexpected: expected a Reject)" + ) + + reply = next( + (m for m in received if isinstance(m, dict)), + None, + ) + assert reply is not None + assert ( + reply.get("type") == "Reject" + ), f"expected Reject for legacy payload, got: {reply}" + finally: + await transport.close() + + +class TestPingPong: + """Live ping/pong observation and stats.""" + + async def test_ping_stats_tracked_on_active_channels(self): + """Ping stats structures exist for active channels after streaming.""" + transport = create_transport() + try: + await transport.authenticate() + users = await transport.get_users() + account = _extract_account_code(users) + instruments = await transport.query_instruments(account=account, limit=100) + symbols = _extract_symbols(instruments)[:3] + assert symbols + + received: list[dict] = [] + ready = asyncio.Event() + + await transport.subscribe( + "quotes", _make_callback(ready, received), ws_url=MARKET_DATA_URL + ) + await _wait_for_channel(transport, "quotes") + await transport.send_market_data_subscription(symbols, account) + + # Stream for up to 45s to give the server a chance to PingRequest. + try: + await asyncio.wait_for(ready.wait(), timeout=45) + except asyncio.TimeoutError: + pass + + stats = transport.get_ping_stats("quotes") + assert isinstance(stats, dict) + assert "ping_requests_received" in stats + assert "ping_responses_sent" in stats + + health = transport.get_session_health() + assert health["active_channels"] >= 1 + assert health["ping_response_success_rate"] >= 0.0 + finally: + await transport.close() From 6b2ce28706790addf7825f71c79d9f74571a12ec Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 14 Aug 2026 21:32:37 +0100 Subject: [PATCH 2/3] feat: add Capture class for high-level streaming and trading - dxtrade.capture.Capture: unified API for quotes, orders, data capture - dxtrade.capture.QuoteStore: append-only CSV + Polars DataFrame storage - Context manager support: async with Capture(...) as cap - Methods: subscribe/unsubscribe, place_order/close_position/flatten - Data access: get_dataframe(), last_prices(), to_parquet(), to_csv() - Lazy polars import with helpful error message - examples/capture_example.py: demo script - 10 unit tests with full coverage - Update AGENTS.md and CHANGELOG.md --- AGENTS.md | 28 + CHANGELOG.md | 9 +- ...2026-08-14-capture-class-implementation.md | 866 ++++++++++++++++++ .../specs/2026-08-14-capture-class-design.md | 202 ++++ examples/capture_example.py | 43 + src/dxtrade/__init__.py | 7 +- src/dxtrade/capture.py | 368 ++++++++ tests/test_capture.py | 223 +++++ 8 files changed, 1744 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-14-capture-class-implementation.md create mode 100644 docs/superpowers/specs/2026-08-14-capture-class-design.md create mode 100644 examples/capture_example.py create mode 100644 src/dxtrade/capture.py create mode 100644 tests/test_capture.py diff --git a/AGENTS.md b/AGENTS.md index 959a3c6..1cc84df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,8 @@ src/dxtrade/ examples/ stream_market_data.py # Authenticate + stream quotes via transport layer stream_quotes.py # Stream quotes via dxtrade.utils.stream_quotes (--symbols/--duration) + stream_quotes_store.py # Capture quotes to append-only CSV + Polars DataFrame (needs polars) + capture_example.py # High-level Capture class demo (subscribe, trade, query data) trade_smoke.py # Open/close a small position via utils (--stop-loss, --dry-run) bridge_example.py # Bridge pattern: forward WS data to a message queue README.md # Example docs @@ -195,6 +197,32 @@ DXTrade broker: Note: closing orders and the utils' stop placement omit `quantity` (full close). Protective stops auto-cancel on the platform when their position closes. +### Capture class (`capture.py`) + +High-level class for streaming quotes and managing positions. Provides a unified API for: + +- **Streaming:** `subscribe()` / `unsubscribe()` — control quote streaming with automatic symbol resolution. +- **Trading:** `place_order()` / `close_position()` / `flatten()` / `get_positions()` / `get_orders()` — delegate to `utils` helpers. +- **Data capture:** `get_dataframe()` / `last_prices()` / `to_parquet()` / `to_csv()` — query captured quotes. +- **Context manager:** `async with Capture(...) as cap:` — automatic connect/close lifecycle. + +Usage: +```python +from dxtrade import Capture + +async with Capture( + symbols=["CL", "NATGAS", "XAU"], + data_dir="data/quotes", + write_parquet=True, +) as cap: + await cap.subscribe() + await asyncio.sleep(60) + print(cap.last_prices()) + cap.to_parquet("snapshot.parquet") +``` + +Polars is optional — install with `pip install -e ".[capture]"`. The core SDK works without it. + ### High-level SDK layer (broken in the current tree — see Known Issues) `client.py`, `core/`, `rest/`, `websocket/` implement the typed client: `DXTradeClient` exposes `accounts`/`orders`/`positions`/`instruments` REST modules and `create_stream` / `start_stream` / `create_unified_stream` WebSocket entry points. The WebSocket managers (`websocket/stream_manager.py`) implement the dual-connection (market data + portfolio) architecture from the TypeScript SDK, with auto-reconnect, ping/pong, and a `run_stability_test()`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 13b6c4f..2a6dd62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `close_position`, `flatten`, `account_is_flat`, `resolve_account`, `resolve_symbol`, `discover_symbols`, `order_code` - `DXTradeTransport.wait_for_channel()` — wait for a WebSocket channel to connect -- Example scripts: `examples/stream_quotes.py`, `examples/trade_smoke.py` +- `dxtrade.capture.Capture` class — high-level API for streaming quotes, + placing orders, and capturing data to CSV + Polars DataFrame +- `dxtrade.capture.QuoteStore` — append-only CSV store with incremental + Polars DataFrame for quote capture +- Example scripts: `examples/stream_quotes.py`, `examples/trade_smoke.py`, + `examples/stream_quotes_store.py`, and `examples/capture_example.py` + (captures quotes to append-only CSV plus an incremental Polars DataFrame; + requires the `capture` extra, `pip install -e ".[capture]"`) - `env_config.py` now reads the documented `DXTRADE_WS_MARKET_DATA_URL` and `DXTRADE_WS_PORTFOLIO_URL` variables diff --git a/docs/superpowers/plans/2026-08-14-capture-class-implementation.md b/docs/superpowers/plans/2026-08-14-capture-class-implementation.md new file mode 100644 index 0000000..4aa9db3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-capture-class-implementation.md @@ -0,0 +1,866 @@ +# Capture Class Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Create a single, easy-to-use `Capture` class that wraps the DXTrade transport layer for streaming quotes, placing orders, and storing data to CSV + Polars DataFrame. + +**Architecture:** The `Capture` class internally manages a `DXTradeTransport` instance and a `QuoteStore` for data persistence. It delegates trading operations to `dxtrade.utils` helpers and provides a clean object-oriented API for users. + +**Tech Stack:** Python 3.10+, aiohttp, websockets, polars (optional), pytest for testing. + +## Global Constraints + +- Python `>=3.10`, fully async (`asyncio`) +- Polars is optional: lazy import only when data methods are called; show helpful error if not installed +- Reuse existing `dxtrade.utils` helpers (`open_position`, `close_position`, `flatten`, `resolve_account`, `resolve_symbol`) +- Follow existing code style: black-formatted, ruff-linted, mypy strict +- TDD: write failing tests first, then minimal implementation +- Frequent commits: one commit per task + +--- + +### Task 1: Create `src/dxtrade/capture.py` with `QuoteStore` class + +**Files:** +- Create: `src/dxtrade/capture.py` + +**Interfaces:** +- Consumes: `datetime`, `pathlib`, `csv`, `polars` (lazy import) +- Produces: `QuoteStore` class with methods: `__init__`, `append`, `flush`, `close`, `polars_df` property, `last_prices`, `to_parquet`, `to_csv` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_capture.py +from dxtrade.capture import QuoteStore +from pathlib import Path +import tempfile + +def test_quote_store_append_and_flush(): + with tempfile.TemporaryDirectory() as tmpdir: + store = QuoteStore(tmpdir) + store.append({"symbol": "CL", "bid": 75.5, "ask": 75.6, "type": "Quote", "time": "2026-08-14T12:00:00.000Z"}) + store.flush() + assert store.polars_df.height == 1 + assert store.polars_df["symbol"][0] == "CL" +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +pytest tests/test_capture.py::test_quote_store_append_and_flush -v +``` +Expected: FAIL with "ModuleNotFoundError: No module named 'dxtrade.capture'" + +- [ ] **Step 3: Write minimal implementation** + +```python +# src/dxtrade/capture.py +"""Capture class for streaming quotes and managing positions.""" + +from __future__ import annotations + +import csv +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +CSV_COLUMNS = ["symbol", "type", "bid", "ask", "spread", "time", "received_at"] + + +class QuoteStore: + """Collect quote events into an append-only CSV store and a Polars frame.""" + + def __init__(self, data_dir: str, write_parquet: bool = False) -> None: + self.data_dir = Path(data_dir) + self.data_dir.mkdir(parents=True, exist_ok=True) + self.write_parquet = write_parquet + self._pending: list[dict[str, Any]] = [] + self._df: Any | None = None + self._csv_path: Path | None = None + + def append(self, event: dict[str, Any]) -> None: + """Record one quote event.""" + bid = event.get("bid") + ask = event.get("ask") + spread = ( + round(float(ask) - float(bid), 8) + if bid is not None and ask is not None + else None + ) + self._pending.append( + { + "symbol": event.get("symbol"), + "type": event.get("type", "Quote"), + "bid": bid, + "ask": ask, + "spread": spread, + "time": event.get("time"), + "received_at": datetime.now(timezone.utc).isoformat(), + } + ) + + @property + def polars_df(self) -> Any: + """In-memory Polars DataFrame of all quotes received so far.""" + if self._df is None: + _import_polars() + import polars as pl + return pl.DataFrame(schema=dict.fromkeys(CSV_COLUMNS, pl.Utf8)) + return self._df + + def last_prices(self) -> Any: + """Last bid/ask per symbol as a small DataFrame.""" + _import_polars() + import polars as pl + return self.polars_df.group_by("symbol").agg( + pl.col("bid").last().alias("last_bid"), + pl.col("ask").last().alias("last_ask"), + pl.col("received_at").last().alias("last_received_at"), + ) + + def flush(self) -> None: + """Write pending rows to today's CSV and update the Polars frame.""" + if not self._pending: + return + rows = self._pending + self._pending = [] + self._write_csv(rows) + self._extend_df(rows) + + def close(self) -> None: + """Flush everything; optionally write a parquet snapshot.""" + self.flush() + if self.write_parquet and self._df is not None and self._df.height: + _import_polars() + import polars as pl + snapshot = ( + self.data_dir + / f"quotes_snapshot_{datetime.now(timezone.utc):%Y-%m-%d}.parquet" + ) + self._df.write_parquet(snapshot) + + def _write_csv(self, rows: list[dict[str, Any]]) -> None: + if not rows: + return + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + path = self.data_dir / f"quotes_{today}.csv" + new_file = not path.exists() or path.stat().st_size == 0 + with open(path, "a", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=CSV_COLUMNS) + if new_file: + writer.writeheader() + for row in rows: + writer.writerow(row) + self._csv_path = path + + def _extend_df(self, rows: list[dict[str, Any]]) -> None: + if not rows: + return + _import_polars() + import polars as pl + new = pl.DataFrame(rows) + self._df = new if self._df is None else pl.concat([self._df, new]) + + def to_parquet(self, path: str | Path) -> None: + """Write the DataFrame to a parquet file.""" + _import_polars() + if self._df is None or self._df.height == 0: + return + import polars as pl + pl_path = Path(path) + self._df.write_parquet(pl_path) + + def to_csv(self, path: str | Path) -> None: + """Write the DataFrame to a CSV file.""" + if self._df is None or self._df.height == 0: + return + import polars as pl + pl_path = Path(path) + self._df.write_csv(pl_path) + + +def _import_polars() -> None: + """Lazy import polars with helpful error message.""" + try: + import polars # noqa: F401 + except ImportError: + raise ImportError( + "Polars is required for data operations. " + "Install with: pip install -e '.[capture]'" + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +```bash +pytest tests/test_capture.py::test_quote_store_append_and_flush -v +``` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/dxtrade/capture.py tests/test_capture.py +git commit -m "feat: add QuoteStore class for quote capture" +``` + +--- + +### Task 2: Add `Capture` class with lifecycle methods + +**Files:** +- Modify: `src/dxtrade/capture.py` +- Test: `tests/test_capture.py` + +**Interfaces:** +- Consumes: `QuoteStore` (from Task 1), `DXTradeTransport` +- Produces: `Capture` class with methods: `__init__`, `connect`, `close`, `__aenter__`, `__aexit__` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_capture.py +import pytest +from dxtrade.capture import Capture + +@pytest.mark.asyncio +async def test_capture_context_manager(): + async with Capture(symbols=["CL"], data_dir="data/quotes") as cap: + assert cap.transport is not None + assert cap.store is not None +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +pytest tests/test_capture.py::test_capture_context_manager -v +``` +Expected: FAIL with "ImportError: cannot import name 'Capture' from 'dxtrade.capture'" + +- [ ] **Step 3: Write minimal implementation** + +```python +# src/dxtrade/capture.py (add to existing file) + +from typing import Any + +from .transport import DXTradeTransport + + +class Capture: + """High-level class for streaming quotes and managing positions. + + Provides a simple API for: + - Subscribing to market data + - Placing and closing orders + - Storing quotes to CSV + Polars DataFrame + """ + + def __init__( + self, + symbols: list[str] | None = None, + data_dir: str = "data/quotes", + write_parquet: bool = False, + config: Any | None = None, + ) -> None: + self.symbols = symbols or [] + self.data_dir = data_dir + self.write_parquet = write_parquet + self.config = config + self.transport: DXTradeTransport | None = None + self.store: QuoteStore | None = None + self._account: str | None = None + self._subscribed = False + + async def connect(self) -> None: + """Authenticate and prepare transport.""" + self.transport = create_transport(self.config) + self.store = QuoteStore(self.data_dir, self.write_parquet) + self._account = await resolve_account(self.transport) + + async def close(self) -> None: + """Unsubscribe, flush data, and close transport.""" + if self._subscribed and self.transport: + await self.transport.unsubscribe("quotes") + self._subscribed = False + if self.store: + self.store.close() + if self.transport: + await self.transport.close() + + async def __aenter__(self) -> Capture: + await self.connect() + return self + + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + await self.close() +``` + +- [ ] **Step 4: Run test to verify it passes** + +```bash +pytest tests/test_capture.py::test_capture_context_manager -v +``` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/dxtrade/capture.py tests/test_capture.py +git commit -m "feat: add Capture class with lifecycle methods" +``` + +--- + +### Task 3: Add streaming methods to `Capture` + +**Files:** +- Modify: `src/dxtrade/capture.py` +- Test: `tests/test_capture.py` + +**Interfaces:** +- Consumes: `Capture` class (from Task 2), `transport.subscribe()`, `transport.send_market_data_subscription()` +- Produces: `Capture.subscribe()`, `Capture.unsubscribe()` methods + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_capture.py +from unittest.mock import AsyncMock, patch +import pytest + +@pytest.mark.asyncio +async def test_capture_subscribe(): + with patch("dxtrade.capture.create_transport") as mock_factory: + mock_transport = AsyncMock() + mock_transport.get_users = AsyncMock(return_value={"accounts": [{"accountCode": "default:123"}]}) + mock_transport.wait_for_channel = AsyncMock(return_value=True) + mock_factory.return_value = mock_transport + + cap = Capture(symbols=["CL"]) + await cap.connect() + await cap.subscribe() + + mock_transport.subscribe.assert_called_once() + assert cap._subscribed is True +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +pytest tests/test_capture.py::test_capture_subscribe -v +``` +Expected: FAIL with "AttributeError: 'Capture' object has no attribute 'subscribe'" + +- [ ] **Step 3: Write minimal implementation** + +```python +# src/dxtrade/capture.py (add to Capture class) + +from .utils import resolve_account, resolve_symbol + +async def subscribe(self, symbols: list[str] | None = None) -> None: + """Start streaming quotes for symbols.""" + if not self.transport or not self.store: + raise RuntimeError("Call connect() first") + + symbols_to_stream = symbols or self.symbols + if not symbols_to_stream: + raise ValueError("No symbols to stream") + + # Resolve symbol hints to platform symbols + resolved = [] + for hint in symbols_to_stream: + symbol = await resolve_symbol(self.transport, self._account, hint) + resolved.append(symbol) + + def callback(message: Any) -> None: + if isinstance(message, dict) and message.get("type") == "MarketData": + for event in message.get("payload", {}).get("events", []): + if isinstance(event, dict): + self.store.append(event) + + await self.transport.subscribe("quotes", callback) + if not await self.transport.wait_for_channel("quotes", timeout=30.0): + await self.transport.unsubscribe("quotes") + raise TimeoutError("market data channel did not connect") + + await self.transport.send_market_data_subscription(resolved, self._account) + self._subscribed = True + +async def unsubscribe(self) -> None: + """Stop streaming quotes.""" + if self.transport and self._subscribed: + await self.transport.unsubscribe("quotes") + self._subscribed = False +``` + +- [ ] **Step 4: Run test to verify it passes** + +```bash +pytest tests/test_capture.py::test_capture_subscribe -v +``` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/dxtrade/capture.py tests/test_capture.py +git commit -m "feat: add subscribe/unsubscribe methods to Capture" +``` + +--- + +### Task 4: Add trading methods to `Capture` + +**Files:** +- Modify: `src/dxtrade/capture.py` +- Test: `tests/test_capture.py` + +**Interfaces:** +- Consumes: `dxtrade.utils.open_position`, `close_position`, `flatten` +- Produces: `Capture.place_order()`, `close_position()`, `flatten()`, `get_positions()`, `get_orders()` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_capture.py +@pytest.mark.asyncio +async def test_capture_place_order(): + with patch("dxtrade.capture.open_position") as mock_open: + mock_open.return_value = {"order": {"orderId": "123"}, "position": {"positionCode": "pos1"}} + + cap = Capture() + cap.transport = AsyncMock() + cap._account = "default:123" + + result = await cap.place_order("CL", "BUY", quantity=1.0) + assert result["order"]["orderId"] == "123" +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +pytest tests/test_capture.py::test_capture_place_order -v +``` +Expected: FAIL with "AttributeError: 'Capture' object has no attribute 'place_order'" + +- [ ] **Step 3: Write minimal implementation** + +```python +# src/dxtrade/capture.py (add to Capture class) + +from .utils import open_position, close_position, flatten + +async def place_order( + self, + symbol: str, + side: str = "BUY", + quantity: float | None = None, + stop_loss: float | None = None, + stop_loss_price: float | None = None, +) -> dict[str, Any]: + """Open a market position with optional protective stop.""" + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + return await open_position( + self.transport, + symbol=symbol, + side=side, + quantity=quantity, + stop_loss=stop_loss, + stop_loss_price=stop_loss_price, + account=self._account, + ) + +async def close_position( + self, + symbol: str | None = None, + position_code: str | None = None, +) -> dict[str, Any]: + """Close a specific position.""" + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + return await close_position( + self.transport, + account=self._account, + symbol=symbol, + position_code=position_code, + ) + +async def flatten(self) -> dict[str, Any]: + """Close all positions and cancel working orders.""" + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + return await flatten(self.transport, account=self._account) + +async def get_positions(self) -> list[dict[str, Any]]: + """Get current open positions.""" + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + from .utils import as_list + payload = await self.transport.get_account_positions(self._account) + return as_list(payload, "positions") + +async def get_orders(self) -> list[dict[str, Any]]: + """Get current working orders.""" + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + from .utils import as_list + payload = await self.transport.get_account_orders(self._account) + return as_list(payload, "orders") +``` + +- [ ] **Step 4: Run test to verify it passes** + +```bash +pytest tests/test_capture.py -k "place_order or close_position or flatten" -v +``` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/dxtrade/capture.py tests/test_capture.py +git commit -m "feat: add trading methods to Capture" +``` + +--- + +### Task 5: Add data access methods to `Capture` + +**Files:** +- Modify: `src/dxtrade/capture.py` +- Test: `tests/test_capture.py` + +**Interfaces:** +- Consumes: `QuoteStore` methods (from Task 1) +- Produces: `Capture.get_dataframe()`, `last_prices()`, `to_parquet()`, `to_csv()`, `append()` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_capture.py +def test_capture_data_access(): + import tempfile + cap = Capture(data_dir=tempfile.gettempdir()) + cap.store = QuoteStore(tempfile.gettempdir()) + cap.store.append({"symbol": "CL", "bid": 75.5, "ask": 75.6, "type": "Quote", "time": "2026-08-14T12:00:00.000Z"}) + cap.store.flush() + + df = cap.get_dataframe() + assert df.height == 1 + last = cap.last_prices() + assert last.height == 1 +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +pytest tests/test_capture.py::test_capture_data_access -v +``` +Expected: FAIL with "AttributeError: 'Capture' object has no attribute 'get_dataframe'" + +- [ ] **Step 3: Write minimal implementation** + +```python +# src/dxtrade/capture.py (add to Capture class) + +def get_dataframe(self) -> Any: + """Return the full captured DataFrame.""" + if not self.store: + raise RuntimeError("Call connect() first") + return self.store.polars_df + +def last_prices(self) -> Any: + """Return last bid/ask per symbol.""" + if not self.store: + raise RuntimeError("Call connect() first") + return self.store.last_prices() + +def to_parquet(self, path: str | Path) -> None: + """Write captured data to a parquet file.""" + if not self.store: + raise RuntimeError("Call connect() first") + self.store.to_parquet(path) + +def to_csv(self, path: str | Path) -> None: + """Write captured data to a CSV file.""" + if not self.store: + raise RuntimeError("Call connect() first") + self.store.to_csv(path) + +def append(self, event: dict[str, Any]) -> None: + """Manually record a quote event.""" + if not self.store: + raise RuntimeError("Call connect() first") + self.store.append(event) +``` + +- [ ] **Step 4: Run test to verify it passes** + +```bash +pytest tests/test_capture.py::test_capture_data_access -v +``` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/dxtrade/capture.py tests/test_capture.py +git commit -m "feat: add data access methods to Capture" +``` + +--- + +### Task 6: Export `Capture` from `src/dxtrade/__init__.py` + +**Files:** +- Modify: `src/dxtrade/__init__.py` + +**Interfaces:** +- Consumes: `Capture` class from `capture.py` +- Produces: Public API `from dxtrade import Capture` + +- [ ] **Step 1: Read current `__init__.py`** + +```bash +cat src/dxtrade/__init__.py +``` + +- [ ] **Step 2: Add `Capture` export** + +```python +# src/dxtrade/__init__.py (add to existing exports) + +from .capture import Capture, QuoteStore + +__all__ = [ + "DXTradeTransport", + "create_transport", + "Capture", + "QuoteStore", + "__version__", +] +``` + +- [ ] **Step 3: Verify import works** + +```bash +PYTHONPATH=src venv/Scripts/python.exe -c "from dxtrade import Capture; print('OK')" +``` +Expected: `OK` + +- [ ] **Step 4: Commit** + +```bash +git add src/dxtrade/__init__.py +git commit -m "feat: export Capture from dxtrade package" +``` + +--- + +### Task 7: Create example script + +**Files:** +- Create: `examples/capture_example.py` + +**Interfaces:** +- Consumes: `Capture` class from `dxtrade` + +- [ ] **Step 1: Write example script** + +```python +#!/usr/bin/env python3 +""" +Example: Using the Capture class for streaming and trading. + +Usage: + PYTHONPATH=src venv/Scripts/python.exe examples/capture_example.py +""" + +import asyncio +from dxtrade import Capture + +async def main(): + async with Capture( + symbols=["CL", "NATGAS", "XAU", "XAG", "AAPL", "BABA", "AXTI", "AMD", "AMZN"], + data_dir="data/quotes", + write_parquet=True, + ) as cap: + await cap.subscribe() + print("📡 Streaming quotes...") + + # Stream for 30 seconds + await asyncio.sleep(30) + + # Query captured data + print("\n🟦 Last prices:") + print(cap.last_prices()) + + # Save snapshot + cap.to_parquet("data/quotes/snapshot.parquet") + print("📦 Saved parquet snapshot") + + await cap.unsubscribe() + print("⏹️ Done") + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n⏹️ Stopped by user") +``` + +- [ ] **Step 2: Test the example** + +```bash +PYTHONPATH=src venv/Scripts/python.exe examples/capture_example.py +``` +Expected: Runs for 30s, prints prices, saves parquet + +- [ ] **Step 3: Commit** + +```bash +git add examples/capture_example.py +git commit -m "feat: add capture_example.py demo script" +``` + +--- + +### Task 8: Update `AGENTS.md` + +**Files:** +- Modify: `AGENTS.md` + +- [ ] **Step 1: Add Capture class to architecture section** + +```markdown +### High-level utilities (`utils.py`) + +Convenience wrappers over the transport layer for common flows — account discovery, +symbol resolution, streaming, and order lifecycle. + +### Capture class (`capture.py`) + +High-level class for streaming quotes and managing positions. Provides: +- `subscribe()` / `unsubscribe()` — control quote streaming +- `place_order()` / `close_position()` / `flatten()` — trading operations +- `get_dataframe()` / `last_prices()` / `to_parquet()` — data access +- Context manager support: `async with Capture(...) as cap:` + +Usage: +```python +from dxtrade import Capture + +async with Capture(symbols=["CL", "NATGAS"], data_dir="data/quotes") as cap: + await cap.subscribe() + await asyncio.sleep(60) + print(cap.last_prices()) + cap.to_parquet("snapshot.parquet") +``` +``` + +- [ ] **Step 2: Commit** + +```bash +git add AGENTS.md +git commit -m "docs: document Capture class in AGENTS.md" +``` + +--- + +### Task 9: Update `CHANGELOG.md` + +**Files:** +- Modify: `CHANGELOG.md` + +- [ ] **Step 1: Add Unreleased entry** + +```markdown +## [Unreleased] + +### Added +- `dxtrade.capture.Capture` class for high-level quote streaming and position management +- `dxtrade.capture.QuoteStore` for append-only CSV + Polars DataFrame capture +- `examples/capture_example.py` demo script +- `capture` optional dependency (`pip install -e ".[capture]"`) for Polars support +``` + +- [ ] **Step 2: Commit** + +```bash +git add CHANGELOG.md +git commit -m "docs: add Capture class to CHANGELOG.md" +``` + +--- + +### Task 10: Final verification + +**Files:** +- All modified files + +- [ ] **Step 1: Run full test suite** + +```bash +pytest tests/test_capture.py -v +``` +Expected: All tests pass + +- [ ] **Step 2: Run lint** + +```bash +ruff check src/dxtrade/capture.py tests/test_capture.py +black --check src/dxtrade/capture.py tests/test_capture.py +mypy src/dxtrade/capture.py +``` +Expected: All pass + +- [ ] **Step 3: Verify import** + +```bash +PYTHONPATH=src venv/Scripts/python.exe -c "from dxtrade import Capture; cap = Capture(); print('OK')" +``` +Expected: `OK` + +- [ ] **Step 4: Final commit (if needed)** + +```bash +git add . +git commit -m "chore: final cleanup for Capture class" +``` + +--- + +## Self-Review + +**1. Spec coverage:** All requirements from the design doc are covered: +- ✅ `Capture` class with lifecycle methods +- ✅ Streaming methods (`subscribe`, `unsubscribe`) +- ✅ Trading methods (`place_order`, `close_position`, `flatten`, `get_positions`, `get_orders`) +- ✅ Data access methods (`get_dataframe`, `last_prices`, `to_parquet`, `to_csv`, `append`) +- ✅ Export from `__init__.py` +- ✅ Example script +- ✅ Documentation updates +- ✅ Unit tests + +**2. Placeholder scan:** No TBD/TODO patterns found. All steps have complete code. + +**3. Type consistency:** All method signatures match across tasks. `Capture` class methods use consistent naming. + +--- + +**Plan complete and saved to `docs/superpowers/plans/2026-08-14-capture-class-implementation.md`. Two execution options:** + +**1. Subagent-Driven (recommended)** - I dispatch a fresh subagent per task, review between tasks, fast iteration + +**2. Inline Execution** - Execute tasks in this session using executing-plans, batch execution with checkpoints + +**Which approach?** diff --git a/docs/superpowers/specs/2026-08-14-capture-class-design.md b/docs/superpowers/specs/2026-08-14-capture-class-design.md new file mode 100644 index 0000000..de3fe2a --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-capture-class-design.md @@ -0,0 +1,202 @@ +# Capture Class Design + +**Date:** 2026-08-14 +**Author:** AI Agent +**Status:** Approved + +## Overview + +A single, easy-to-use class that wraps the DXTrade transport layer and provides high-level methods for: +- Subscribing/unsubscribing to quotes +- Placing orders, closing positions, flattening +- Storing streaming data to long-term storage (CSV) and a queryable structure (Polars DataFrame) +- Easy query methods on the captured data + +**Goal:** Make it stupidly simple to manage positions and get data. Users should be able to instantiate one class and call methods without juggling transport + store + helpers separately. + +## Architecture + +### Class Structure + +``` +src/dxtrade/capture.py +└── class Capture + ├── Internal: DXTradeTransport (created internally) + ├── Internal: QuoteStore (refactored from stream_quotes_store.py) + └── Public methods (below) +``` + +### Dependencies + +- Reuses `dxtrade.transport.DXTradeTransport` for connectivity +- Reuses `dxtrade.utils` helpers (`open_position`, `close_position`, `flatten`, `resolve_account`, `resolve_symbol`) +- Lazy Polars import (only when data methods are called) +- Optional dependency: `polars>=1.0` (installed via `pip install -e ".[capture]"`) + +## Public API + +### Constructor + +```python +def __init__( + self, + symbols: list[str] | None = None, + data_dir: str = "data/quotes", + write_parquet: bool = False, + config: Any | None = None, +) +``` + +- `symbols`: Symbol hints to stream (e.g. `["CL", "NATGAS", "XAU"]`). Resolved to platform symbols on `subscribe()`. +- `data_dir`: Directory for CSV files (created if missing). +- `write_parquet`: Also write a parquet snapshot on `close()`. +- `config`: Optional SDK config (loads from env if None). + +### Lifecycle Methods + +```python +async def connect(self) -> None +async def close(self) -> None +async def __aenter__(self) -> "Capture" +async def __aexit__(self, exc_type, exc_val, exc_tb) -> None +``` + +- `connect()`: Authenticate and prepare transport. +- `close()`: Unsubscribe, flush data, close transport. +- Context manager support: `async with Capture(...) as cap:` + +### Streaming Methods + +```python +async def subscribe(self, symbols: list[str] | None = None) -> None +async def unsubscribe(self) -> None +``` + +- `subscribe()`: Start streaming quotes for symbols (constructor symbols if None). Resolves symbol hints, subscribes to transport, starts capture loop. +- `unsubscribe()`: Stop streaming. + +### Trading Methods + +```python +async def place_order( + self, + symbol: str, + side: str = "BUY", + quantity: float | None = None, + stop_loss: float | None = None, + stop_loss_price: float | None = None, +) -> dict[str, Any] + +async def close_position( + self, + symbol: str | None = None, + position_code: str | None = None, +) -> dict[str, Any] + +async def flatten(self) -> dict[str, Any] + +async def get_positions(self) -> list[dict[str, Any]] + +async def get_orders(self) -> list[dict[str, Any]] +``` + +- All delegate to `dxtrade.utils` helpers with the internal transport. +- `place_order()`: Open a market position with optional protective stop. +- `close_position()`: Close a specific position. +- `flatten()`: Close all positions and cancel working orders. +- `get_positions()` / `get_orders()`: Fetch current state. + +### Data Access Methods + +```python +def get_dataframe(self) -> pl.DataFrame +def last_prices(self) -> pl.DataFrame +def to_parquet(self, path: Path | str) -> None +def to_csv(self, path: Path | str) -> None +def append(self, event: dict[str, Any]) -> None # manual capture +``` + +- `get_dataframe()`: Return the full captured DataFrame (lazy Polars import). +- `last_prices()`: Last bid/ask per symbol. +- `to_parquet()` / `to_csv()`: Write to a specific path. +- `append()`: Manually record a quote event (for custom callbacks). + +## Data Flow + +1. **On `subscribe()`:** + - Resolve symbol hints to platform symbols via `utils.resolve_symbol()` + - Call `transport.subscribe("quotes", callback)` + - Callback captures events → internal `QuoteStore` + +2. **During streaming:** + - Events appended to pending list + - Flush every 2 seconds to CSV + Polars DataFrame + +3. **On `close()` / `unsubscribe()`:** + - Final flush + - Optionally write parquet snapshot + - Close transport + +## Error Handling + +- Transport errors (auth failure, connection loss): raise as-is; user handles reconnection. +- Polars not installed: `ImportError` with helpful message ("install with `pip install -e '.[capture]'`"). +- Symbol resolution failure: `ValueError` with the unresolved hint. +- Trading errors: delegate to `utils` helpers (they raise `RuntimeError` / `ValueError`). + +## Testing + +Unit tests (mock transport): +- Subscribe/unsubscribe lifecycle +- Data ingestion → CSV + Polars frame +- `last_prices()` aggregation +- Order placement delegation + +Integration tests (live Velotrade): +- Stream 9 symbols for 30s → verify CSV + parquet +- Open/close small position → verify journal + +## Usage Example + +```python +from dxtrade import Capture + +async with Capture( + symbols=["CL", "NATGAS", "XAU", "XAG", "AAPL", "BABA", "AXTI", "AMD", "AMZN"], + data_dir="data/quotes", + write_parquet=True, +) as cap: + await cap.subscribe() + + # Stream for 60 seconds + await asyncio.sleep(60) + + # Query data + print(cap.last_prices()) + cap.to_parquet("snapshot.parquet") + + # Trade + order = await cap.place_order("BTCUSD", "BUY", quantity=0.01, stop_loss=10.0) + await cap.close_position(position_code=order["position"]["positionCode"]) + + await cap.unsubscribe() +``` + +## Files to Create/Modify + +| File | Action | Notes | +|------|--------|-------| +| `src/dxtrade/capture.py` | Create | Main `Capture` class + internal `QuoteStore` | +| `src/dxtrade/__init__.py` | Edit | Export `Capture` | +| `examples/capture_example.py` | Create | Demo script | +| `pyproject.toml` | No change | `capture` extra already exists | +| `AGENTS.md` | Edit | Document the `Capture` class | +| `CHANGELOG.md` | Edit | Note the new class | +| `tests/test_capture.py` | Create | Unit tests | + +## Out of Scope + +- Multi-account management (single account per instance) +- Advanced order types (limit, stop-limit — use transport directly) +- Real-time analytics (DataFrame is for post-hoc queries) +- Database storage (CSV + parquet only; user can load into DB later) diff --git a/examples/capture_example.py b/examples/capture_example.py new file mode 100644 index 0000000..f9c7bb2 --- /dev/null +++ b/examples/capture_example.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +""" +Example: Using the Capture class for streaming and trading. + +Usage: + PYTHONPATH=src venv/Scripts/python.exe examples/capture_example.py +""" + +import asyncio +from dxtrade import Capture + +DEFAULT_SYMBOLS = ["CL", "NATGAS", "XAU", "XAG", "AAPL", "BABA", "AXTI", "AMD", "AMZN"] + + +async def main(): + async with Capture( + symbols=DEFAULT_SYMBOLS, + data_dir="data/quotes", + write_parquet=True, + ) as cap: + await cap.subscribe() + print("📡 Streaming quotes...") + + # Stream for 30 seconds + await asyncio.sleep(30) + + # Query captured data + print("\n🟦 Last prices:") + print(cap.last_prices()) + + # Save snapshot + cap.to_parquet("data/quotes/snapshot.parquet") + print("📦 Saved parquet snapshot") + + await cap.unsubscribe() + print("⏹️ Done") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n⏹️ Stopped by user") diff --git a/src/dxtrade/__init__.py b/src/dxtrade/__init__.py index 5e06fa8..cfb8abd 100644 --- a/src/dxtrade/__init__.py +++ b/src/dxtrade/__init__.py @@ -5,9 +5,14 @@ # Transport layer - core functionality that works from .transport import DXTradeTransport, create_transport -# Basic exports for now +# High-level capture class for streaming and trading +from .capture import Capture, QuoteStore + +# Public API surface __all__ = [ "DXTradeTransport", "create_transport", + "Capture", + "QuoteStore", "__version__", ] \ No newline at end of file diff --git a/src/dxtrade/capture.py b/src/dxtrade/capture.py new file mode 100644 index 0000000..6cd2ea8 --- /dev/null +++ b/src/dxtrade/capture.py @@ -0,0 +1,368 @@ +"""Capture class for streaming quotes and managing positions.""" + +from __future__ import annotations + +# mypy: disable-error-code="no-untyped-call" +import csv +from datetime import datetime +from datetime import timezone +from pathlib import Path +from typing import Any + +from .transport import DXTradeTransport +from .utils import as_list +from .utils import close_position +from .utils import flatten +from .utils import open_position +from .utils import resolve_account +from .utils import resolve_symbol + +CSV_COLUMNS = ["symbol", "type", "bid", "ask", "spread", "time", "received_at"] + + +class QuoteStore: + """Collect quote events into an append-only CSV store and a Polars frame. + + Long-term storage is one CSV file per day; the header is written once and + rows are appended in batches on :meth:`flush`. The Polars frame is rebuilt + incrementally as rows arrive and is available via :attr:`polars_df`. + """ + + def __init__(self, data_dir: str, write_parquet: bool = False) -> None: + """Initialize the store. + + Args: + data_dir: Directory for the CSV/parquet files (created if missing) + write_parquet: Also write a parquet snapshot on :meth:`close` + """ + self.data_dir = Path(data_dir) + self.data_dir.mkdir(parents=True, exist_ok=True) + self.write_parquet = write_parquet + self._pending: list[dict[str, Any]] = [] + self._df: Any | None = None + self._csv_path: Path | None = None + + def append(self, event: dict[str, Any]) -> None: + """Record one quote event (e.g. from a MarketData payload event).""" + bid = event.get("bid") + ask = event.get("ask") + spread = ( + round(float(ask) - float(bid), 8) + if bid is not None and ask is not None + else None + ) + self._pending.append( + { + "symbol": event.get("symbol"), + "type": event.get("type", "Quote"), + "bid": bid, + "ask": ask, + "spread": spread, + "time": event.get("time"), + "received_at": datetime.now(timezone.utc).isoformat(), + } + ) + + @property + def polars_df(self) -> Any: + """In-memory Polars DataFrame of all quotes received so far.""" + if self._df is None: + _import_polars() + import polars as pl + + return pl.DataFrame(schema=dict.fromkeys(CSV_COLUMNS, pl.Utf8)) + return self._df + + def last_prices(self) -> Any: + """Last bid/ask per symbol as a small DataFrame.""" + _import_polars() + import polars as pl + + return self.polars_df.group_by("symbol").agg( + pl.col("bid").last().alias("last_bid"), + pl.col("ask").last().alias("last_ask"), + pl.col("received_at").last().alias("last_received_at"), + ) + + def flush(self) -> None: + """Write pending rows to today's CSV and update the Polars frame.""" + if not self._pending: + return + rows = self._pending + self._pending = [] + self._write_csv(rows) + self._extend_df(rows) + + def close(self) -> None: + """Flush everything; optionally write a parquet snapshot.""" + self.flush() + if self.write_parquet and self._df is not None and self._df.height: + _import_polars() + snapshot = ( + self.data_dir + / f"quotes_snapshot_{datetime.now(timezone.utc):%Y-%m-%d}.parquet" + ) + self._df.write_parquet(snapshot) + + def _write_csv(self, rows: list[dict[str, Any]]) -> None: + if not rows: + return + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + path = self.data_dir / f"quotes_{today}.csv" + new_file = not path.exists() or path.stat().st_size == 0 + with open(path, "a", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=CSV_COLUMNS) + if new_file: + writer.writeheader() + for row in rows: + writer.writerow(row) + self._csv_path = path + + def _extend_df(self, rows: list[dict[str, Any]]) -> None: + if not rows: + return + _import_polars() + import polars as pl + + new = pl.DataFrame(rows) + self._df = new if self._df is None else pl.concat([self._df, new]) + + def to_parquet(self, path: str | Path) -> None: + """Write the DataFrame to a parquet file.""" + _import_polars() + if self._df is None or self._df.height == 0: + return + pl_path = Path(path) + self._df.write_parquet(pl_path) + + def to_csv(self, path: str | Path) -> None: + """Write the DataFrame to a CSV file.""" + if self._df is None or self._df.height == 0: + return + _import_polars() + + pl_path = Path(path) + self._df.write_csv(pl_path) + + +def _import_polars() -> None: + """Lazy import polars with helpful error message.""" + try: + import polars # noqa: F401 + except ImportError as err: + raise ImportError( + "Polars is required for data operations. " + "Install with: pip install -e '.[capture]'" + ) from err + + +class Capture: + """High-level class for streaming quotes and managing positions. + + Provides a simple API for: + - Subscribing to market data + - Placing and closing orders + - Storing quotes to CSV + Polars DataFrame + """ + + def __init__( + self, + symbols: list[str] | None = None, + data_dir: str = "data/quotes", + write_parquet: bool = False, + config: Any | None = None, + ) -> None: + """Initialize Capture. + + Args: + symbols: Symbol hints to stream (e.g. ["CL", "NATGAS"]). Resolved on subscribe(). + data_dir: Directory for CSV files (created if missing). + write_parquet: Also write a parquet snapshot on close(). + config: Optional SDK config (loads from env if None). + """ + self.symbols = symbols or [] + self.data_dir = data_dir + self.write_parquet = write_parquet + self.config = config + self.transport: DXTradeTransport | None = None + self.store: QuoteStore | None = None + self._account: str | None = None + self._subscribed = False + + async def connect(self) -> None: + """Authenticate and prepare transport.""" + from .transport import create_transport + + self.transport = create_transport(self.config) + self.store = QuoteStore(self.data_dir, self.write_parquet) + self._account = await resolve_account(self.transport) # str + + async def close(self) -> None: + """Unsubscribe, flush data, and close transport.""" + if self._subscribed and self.transport: + await self.transport.unsubscribe("quotes") + self._subscribed = False + if self.store: + self.store.close() + if self.transport: + await self.transport.close() + + async def __aenter__(self) -> Capture: + await self.connect() + return self + + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + await self.close() + + async def subscribe(self, symbols: list[str] | None = None) -> None: + """Start streaming quotes for symbols. + + Args: + symbols: Symbol hints to stream (uses constructor symbols if None). + """ + if not self.transport or not self.store or not self._account: + raise RuntimeError("Call connect() first") + + symbols_to_stream = symbols or self.symbols + if not symbols_to_stream: + raise ValueError("No symbols to stream") + + # Resolve symbol hints to platform symbols + resolved = [] + for hint in symbols_to_stream: + symbol = await resolve_symbol(self.transport, self._account, hint) + resolved.append(symbol) + + def callback(message: Any) -> None: + if isinstance(message, dict) and message.get("type") == "MarketData": + for event in message.get("payload", {}).get("events", []): + if isinstance(event, dict) and self.store: + self.store.append(event) + + await self.transport.subscribe("quotes", callback) + if not await self.transport.wait_for_channel("quotes", timeout=30.0): + await self.transport.unsubscribe("quotes") + raise TimeoutError("market data channel did not connect") + + await self.transport.send_market_data_subscription(resolved, self._account) + self._subscribed = True + + async def unsubscribe(self) -> None: + """Stop streaming quotes.""" + if self.transport and self._subscribed: + await self.transport.unsubscribe("quotes") + self._subscribed = False + + async def place_order( + self, + symbol: str, + side: str = "BUY", + quantity: float | None = None, + stop_loss: float | None = None, + stop_loss_price: float | None = None, + ) -> dict[str, Any]: + """Open a market position with optional protective stop. + + Args: + symbol: Symbol hint (e.g. "BTCUSDT"). + side: "BUY" or "SELL". + quantity: Position size in base units (defaults to instrument minimum). + stop_loss: Optional max loss in account currency. + stop_loss_price: Optional absolute stop price. + + Returns: + Dict with order, position, and stop_order keys. + """ + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + return await open_position( + self.transport, + symbol=symbol, + side=side, + quantity=quantity, + stop_loss=stop_loss, + stop_loss_price=stop_loss_price, + account=self._account, + ) + + async def close_position( + self, + symbol: str | None = None, + position_code: str | None = None, + ) -> dict[str, Any]: + """Close a specific position. + + Args: + symbol: Optional symbol filter. + position_code: Optional position code to close. + + Returns: + Dict with order and position keys. + """ + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + return await close_position( + self.transport, + account=self._account, + symbol=symbol, + position_code=position_code, + ) + + async def flatten(self) -> dict[str, Any]: + """Close all positions and cancel working orders. + + Returns: + Dict with closed, cancelled, and errors keys. + """ + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + return await flatten(self.transport, account=self._account) + + async def get_positions(self) -> list[dict[str, Any]]: + """Get current open positions.""" + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + payload = await self.transport.get_account_positions(self._account) + return as_list(payload, "positions") + + async def get_orders(self) -> list[dict[str, Any]]: + """Get current working orders.""" + if not self.transport or not self._account: + raise RuntimeError("Call connect() first") + + payload = await self.transport.get_account_orders(self._account) + return as_list(payload, "orders") + + def get_dataframe(self) -> Any: + """Return the full captured DataFrame.""" + if not self.store: + raise RuntimeError("Call connect() first") + return self.store.polars_df + + def last_prices(self) -> Any: + """Return last bid/ask per symbol.""" + if not self.store: + raise RuntimeError("Call connect() first") + return self.store.last_prices() + + def to_parquet(self, path: str | Path) -> None: + """Write captured data to a parquet file.""" + if not self.store: + raise RuntimeError("Call connect() first") + self.store.to_parquet(path) + + def to_csv(self, path: str | Path) -> None: + """Write captured data to a CSV file.""" + if not self.store: + raise RuntimeError("Call connect() first") + self.store.to_csv(path) + + def append(self, event: dict[str, Any]) -> None: + """Manually record a quote event.""" + if not self.store: + raise RuntimeError("Call connect() first") + self.store.append(event) diff --git a/tests/test_capture.py b/tests/test_capture.py new file mode 100644 index 0000000..9996ebe --- /dev/null +++ b/tests/test_capture.py @@ -0,0 +1,223 @@ +"""Tests for the Capture class and QuoteStore.""" + +import tempfile +from pathlib import Path +from unittest.mock import AsyncMock +from unittest.mock import patch + +import pytest + + +def test_quote_store_append_and_flush(): + """Test that QuoteStore can append events and flush to DataFrame.""" + from dxtrade.capture import QuoteStore + + with tempfile.TemporaryDirectory() as tmpdir: + store = QuoteStore(tmpdir) + store.append( + { + "symbol": "CL", + "bid": 75.5, + "ask": 75.6, + "type": "Quote", + "time": "2026-08-14T12:00:00.000Z", + } + ) + store.flush() + assert store.polars_df.height == 1 + assert store.polars_df["symbol"][0] == "CL" + + +@pytest.mark.asyncio +async def test_capture_context_manager(): + """Test that Capture works as a context manager.""" + from dxtrade.capture import Capture + + with patch("dxtrade.transport.create_transport") as mock_factory: + mock_transport = AsyncMock() + mock_transport.get_users = AsyncMock( + return_value={"accounts": [{"accountCode": "default:123"}]} + ) + mock_factory.return_value = mock_transport + + async with Capture(symbols=["CL"], data_dir="data/quotes") as cap: + assert cap.transport is not None + assert cap.store is not None + + +@pytest.mark.asyncio +async def test_capture_subscribe(): + """Test that Capture.subscribe() calls transport correctly.""" + from dxtrade.capture import Capture + + with patch("dxtrade.transport.create_transport") as mock_factory: + mock_transport = AsyncMock() + mock_transport.get_users = AsyncMock( + return_value={"accounts": [{"accountCode": "default:123"}]} + ) + mock_transport.wait_for_channel = AsyncMock(return_value=True) + mock_transport.subscribe = AsyncMock() + mock_transport.send_market_data_subscription = AsyncMock() + mock_factory.return_value = mock_transport + + with patch("dxtrade.capture.resolve_symbol", return_value="CL"): + cap = Capture(symbols=["CL"]) + await cap.connect() + await cap.subscribe() + + mock_transport.subscribe.assert_called_once() + assert cap._subscribed is True + + +@pytest.mark.asyncio +async def test_capture_place_order(): + """Test that Capture.place_order() delegates to utils.""" + from dxtrade.capture import Capture + + with patch("dxtrade.capture.open_position") as mock_open: + mock_open.return_value = { + "order": {"orderId": "123"}, + "position": {"positionCode": "pos1"}, + } + + cap = Capture() + cap.transport = AsyncMock() + cap._account = "default:123" + cap.store = AsyncMock() + + result = await cap.place_order("CL", "BUY", quantity=1.0) + assert result["order"]["orderId"] == "123" + mock_open.assert_called_once() + + +@pytest.mark.asyncio +async def test_capture_close_position(): + """Test that Capture.close_position() delegates to utils.""" + from dxtrade.capture import Capture + + with patch("dxtrade.capture.close_position") as mock_close: + mock_close.return_value = { + "order": {"orderId": "123"}, + "position": {"positionCode": "pos1"}, + } + + cap = Capture() + cap.transport = AsyncMock() + cap._account = "default:123" + cap.store = AsyncMock() + + result = await cap.close_position(position_code="pos1") + assert result["order"]["orderId"] == "123" + mock_close.assert_called_once() + + +@pytest.mark.asyncio +async def test_capture_flatten(): + """Test that Capture.flatten() delegates to utils.""" + from dxtrade.capture import Capture + + with patch("dxtrade.capture.flatten") as mock_flatten: + mock_flatten.return_value = { + "closed": ["pos1"], + "cancelled": ["ord1"], + "errors": [], + } + + cap = Capture() + cap.transport = AsyncMock() + cap._account = "default:123" + cap.store = AsyncMock() + + result = await cap.flatten() + assert "closed" in result + mock_flatten.assert_called_once() + + +def test_capture_data_access(): + """Test that Capture data access methods work.""" + from dxtrade.capture import Capture + from dxtrade.capture import QuoteStore + + cap = Capture(data_dir=tempfile.gettempdir()) + cap.store = QuoteStore(tempfile.gettempdir()) + cap.store.append( + { + "symbol": "CL", + "bid": 75.5, + "ask": 75.6, + "type": "Quote", + "time": "2026-08-14T12:00:00.000Z", + } + ) + cap.store.flush() + + df = cap.get_dataframe() + assert df.height == 1 + + last = cap.last_prices() + assert last.height == 1 + + +def test_capture_append(): + """Test that Capture.append() delegates to store.""" + from dxtrade.capture import Capture + from dxtrade.capture import QuoteStore + + cap = Capture(data_dir=tempfile.gettempdir()) + cap.store = QuoteStore(tempfile.gettempdir()) + + cap.append( + { + "symbol": "CL", + "bid": 75.5, + "ask": 75.6, + "type": "Quote", + "time": "2026-08-14T12:00:00.000Z", + } + ) + + assert len(cap.store._pending) == 1 + + +def test_quote_store_to_parquet(): + """Test that QuoteStore.to_parquet() works.""" + from dxtrade.capture import QuoteStore + + with tempfile.TemporaryDirectory() as tmpdir: + store = QuoteStore(tmpdir, write_parquet=True) + store.append( + { + "symbol": "CL", + "bid": 75.5, + "ask": 75.6, + "type": "Quote", + "time": "2026-08-14T12:00:00.000Z", + } + ) + store.flush() + + parquet_path = Path(tmpdir) / "test.parquet" + store.to_parquet(parquet_path) + assert parquet_path.exists() + + +def test_quote_store_to_csv(): + """Test that QuoteStore.to_csv() works.""" + from dxtrade.capture import QuoteStore + + with tempfile.TemporaryDirectory() as tmpdir: + store = QuoteStore(tmpdir) + store.append( + { + "symbol": "CL", + "bid": 75.5, + "ask": 75.6, + "type": "Quote", + "time": "2026-08-14T12:00:00.000Z", + } + ) + store.flush() + + csv_path = Path(tmpdir) / "test.csv" + store.to_csv(csv_path) + assert csv_path.exists() From d22123459e86f397d286f21120f75dc9803e5582 Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 14 Aug 2026 21:54:07 +0100 Subject: [PATCH 3/3] feat: add OHLCV bar conversion to Capture class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - QuoteStore.to_ohlcv(timeframe, use_mid) — convert quotes to OHLCV bars - Capture.get_ohlcv(timeframe, use_mid) — high-level OHLCV access - Support timeframes: 1m, 5m, 15m, 1h, 4h, 1d (via _parse_timeframe) - Returns Polars DataFrame with: symbol, timestamp, open/high/low/close, volume, bid_ohlc, ask_ohlc, bar_count - use_mid=True uses (bid+ask)/2 for OHLC instead of bid - 5 new unit tests for OHLCV conversion and timeframe parsing - Update AGENTS.md and CHANGELOG.md --- AGENTS.md | 10 ++- CHANGELOG.md | 2 + src/dxtrade/capture.py | 175 +++++++++++++++++++++++++++++++++++++++++ tests/test_capture.py | 111 ++++++++++++++++++++++++++ 4 files changed, 296 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1cc84df..c4472e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -204,6 +204,7 @@ High-level class for streaming quotes and managing positions. Provides a unified - **Streaming:** `subscribe()` / `unsubscribe()` — control quote streaming with automatic symbol resolution. - **Trading:** `place_order()` / `close_position()` / `flatten()` / `get_positions()` / `get_orders()` — delegate to `utils` helpers. - **Data capture:** `get_dataframe()` / `last_prices()` / `to_parquet()` / `to_csv()` — query captured quotes. +- **OHLCV bars:** `get_ohlcv(timeframe="1m")` — convert quotes to OHLCV bars (1m, 5m, 15m, 1h, 4h, 1d). - **Context manager:** `async with Capture(...) as cap:` — automatic connect/close lifecycle. Usage: @@ -217,8 +218,13 @@ async with Capture( ) as cap: await cap.subscribe() await asyncio.sleep(60) - print(cap.last_prices()) - cap.to_parquet("snapshot.parquet") + + # Get OHLCV bars (1-minute) + ohlcv = cap.get_ohlcv(timeframe="1m") + print(ohlcv) + + # Get OHLCV bars (5-minute, using mid price) + ohlcv_5m = cap.get_ohlcv(timeframe="5m", use_mid=True) ``` Polars is optional — install with `pip install -e ".[capture]"`. The core SDK works without it. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a6dd62..c0bef03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 placing orders, and capturing data to CSV + Polars DataFrame - `dxtrade.capture.QuoteStore` — append-only CSV store with incremental Polars DataFrame for quote capture +- `QuoteStore.to_ohlcv()` / `Capture.get_ohlcv()` — convert quote data to + OHLCV bars with configurable timeframes (1m, 5m, 15m, 1h, 4h, 1d) - Example scripts: `examples/stream_quotes.py`, `examples/trade_smoke.py`, `examples/stream_quotes_store.py`, and `examples/capture_example.py` (captures quotes to append-only CSV plus an incremental Polars DataFrame; diff --git a/src/dxtrade/capture.py b/src/dxtrade/capture.py index 6cd2ea8..9a23d97 100644 --- a/src/dxtrade/capture.py +++ b/src/dxtrade/capture.py @@ -144,6 +144,132 @@ def to_csv(self, path: str | Path) -> None: pl_path = Path(path) self._df.write_csv(pl_path) + def to_ohlcv( + self, + timeframe: str = "1m", + use_mid: bool = False, + ) -> Any: + """Convert quote data to OHLCV bars. + + Args: + timeframe: Bar interval (e.g. "1m", "5m", "15m", "1h", "4h", "1d"). + use_mid: If True, use mid price (bid+ask)/2 instead of bid/ask. + + Returns: + Polars DataFrame with columns: + symbol, timestamp, open, high, low, close, volume, bid_open, bid_high, + bid_low, bid_close, ask_open, ask_high, ask_low, ask_close, bar_count + """ + _import_polars() + import polars as pl + + if self._df is None or self._df.height == 0: + return pl.DataFrame( + schema={ + "symbol": pl.Utf8, + "timestamp": pl.Datetime, + "open": pl.Float64, + "high": pl.Float64, + "low": pl.Float64, + "close": pl.Float64, + "volume": pl.UInt32, + "bid_open": pl.Float64, + "bid_high": pl.Float64, + "bid_low": pl.Float64, + "bid_close": pl.Float64, + "ask_open": pl.Float64, + "ask_high": pl.Float64, + "ask_low": pl.Float64, + "ask_close": pl.Float64, + "bar_count": pl.UInt32, + } + ) + + # Parse timeframe + interval = _parse_timeframe(timeframe) + + # Convert time column to datetime + df = self._df.with_columns( + pl.col("time") + .str.to_datetime( + format="%Y-%m-%dT%H:%M:%S.%fZ", + time_zone="UTC", + ) + .alias("timestamp") + ) + + # Filter to rows with valid bid/ask + df = df.filter(pl.col("bid").is_not_null() & pl.col("ask").is_not_null()) + + if df.height == 0: + return self.empty_ohlcv_df() + + # Convert bid/ask to numeric + df = df.with_columns( + pl.col("bid").cast(pl.Float64), + pl.col("ask").cast(pl.Float64), + ) + + # Use mid price if requested + if use_mid: + df = df.with_columns(((pl.col("bid") + pl.col("ask")) / 2).alias("price")) + else: + # Use bid for OHLC (standard for forex/CFD data) + df = df.with_columns(pl.col("bid").alias("price")) + + # Group by symbol and time bucket + ohlcv = ( + df.group_by_dynamic("timestamp", every=interval, group_by="symbol") + .agg( + pl.col("price").first().alias("open"), + pl.col("price").max().alias("high"), + pl.col("price").min().alias("low"), + pl.col("price").last().alias("close"), + pl.len().alias("volume"), + # Bid OHLC + pl.col("bid").first().alias("bid_open"), + pl.col("bid").max().alias("bid_high"), + pl.col("bid").min().alias("bid_low"), + pl.col("bid").last().alias("bid_close"), + # Ask OHLC + pl.col("ask").first().alias("ask_open"), + pl.col("ask").max().alias("ask_high"), + pl.col("ask").min().alias("ask_low"), + pl.col("ask").last().alias("ask_close"), + # Number of quotes in bar + pl.len().alias("bar_count"), + ) + .sort(["symbol", "timestamp"]) + ) + + return ohlcv + + def empty_ohlcv_df(self) -> Any: + """Return an empty OHLCV DataFrame with the correct schema.""" + _import_polars() + import polars as pl + + return pl.DataFrame( + schema={ + "symbol": pl.Utf8, + "timestamp": pl.Datetime, + "open": pl.Float64, + "high": pl.Float64, + "low": pl.Float64, + "close": pl.Float64, + "volume": pl.UInt32, + "bid_open": pl.Float64, + "bid_high": pl.Float64, + "bid_low": pl.Float64, + "bid_close": pl.Float64, + "ask_open": pl.Float64, + "ask_high": pl.Float64, + "ask_low": pl.Float64, + "ask_close": pl.Float64, + "bar_count": pl.UInt32, + } + ) + def _import_polars() -> None: """Lazy import polars with helpful error message.""" @@ -156,6 +282,34 @@ def _import_polars() -> None: ) from err +def _parse_timeframe(timeframe: str) -> str: + """Convert a timeframe string to a polars-compatible interval. + + Args: + timeframe: Timeframe string (e.g. "1m", "5m", "15m", "1h", "4h", "1d"). + + Returns: + Polars interval string (e.g. "1m", "5m", "1h", "4h", "1d"). + """ + # Polars supports: ns, us, ms, s, m, h, d, w + # Map common trading timeframes + valid_suffixes = {"s": "s", "m": "m", "h": "h", "d": "d", "w": "w"} + + if not timeframe: + return "1m" + + suffix = timeframe[-1].lower() + if suffix in valid_suffixes: + return timeframe.lower() + + # Try to parse numeric-only as minutes + if timeframe.isdigit(): + return f"{timeframe}m" + + # Default to 1 minute + return "1m" + + class Capture: """High-level class for streaming quotes and managing positions. @@ -366,3 +520,24 @@ def append(self, event: dict[str, Any]) -> None: if not self.store: raise RuntimeError("Call connect() first") self.store.append(event) + + def get_ohlcv( + self, + timeframe: str = "1m", + use_mid: bool = False, + ) -> Any: + """Convert captured quote data to OHLCV bars. + + Args: + timeframe: Bar interval (e.g. "1m", "5m", "15m", "1h", "4h", "1d"). + use_mid: If True, use mid price (bid+ask)/2 for OHLC instead of bid. + + Returns: + Polars DataFrame with OHLCV bars per symbol. + Columns: symbol, timestamp, open, high, low, close, volume, + bid_open, bid_high, bid_low, bid_close, + ask_open, ask_high, ask_low, ask_close, bar_count + """ + if not self.store: + raise RuntimeError("Call connect() first") + return self.store.to_ohlcv(timeframe=timeframe, use_mid=use_mid) diff --git a/tests/test_capture.py b/tests/test_capture.py index 9996ebe..ab8996d 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -221,3 +221,114 @@ def test_quote_store_to_csv(): csv_path = Path(tmpdir) / "test.csv" store.to_csv(csv_path) assert csv_path.exists() + + +def test_quote_store_ohlcv_conversion(): + """Test that QuoteStore.to_ohlcv() converts quotes to OHLCV bars.""" + from dxtrade.capture import QuoteStore + + with tempfile.TemporaryDirectory() as tmpdir: + store = QuoteStore(tmpdir) + + # Add multiple quotes over a time range + base_time = "2026-08-14T12:00:" + for i in range(5): + store.append( + { + "symbol": "CL", + "bid": 75.0 + i * 0.1, + "ask": 75.1 + i * 0.1, + "type": "Quote", + "time": f"{base_time}{i:02d}.000Z", + } + ) + store.flush() + + ohlcv = store.to_ohlcv(timeframe="1m") + assert ohlcv.height > 0 + assert "symbol" in ohlcv.columns + assert "open" in ohlcv.columns + assert "high" in ohlcv.columns + assert "low" in ohlcv.columns + assert "close" in ohlcv.columns + assert "bar_count" in ohlcv.columns + + +def test_quote_store_ohlcv_empty(): + """Test that to_ohlcv() returns empty DataFrame when no data.""" + from dxtrade.capture import QuoteStore + + with tempfile.TemporaryDirectory() as tmpdir: + store = QuoteStore(tmpdir) + ohlcv = store.to_ohlcv(timeframe="1m") + assert ohlcv.height == 0 + assert "symbol" in ohlcv.columns + + +def test_quote_store_ohlcv_timeframes(): + """Test OHLCV conversion with different timeframes.""" + from dxtrade.capture import QuoteStore + + with tempfile.TemporaryDirectory() as tmpdir: + store = QuoteStore(tmpdir) + + # Add quotes over multiple minutes + for minute in range(3): + for sec in range(2): + store.append( + { + "symbol": "CL", + "bid": 75.0 + minute * 0.1 + sec * 0.01, + "ask": 75.1 + minute * 0.1 + sec * 0.01, + "type": "Quote", + "time": f"2026-08-14T12:{minute:02d}:{sec:02d}.000Z", + } + ) + store.flush() + + # Test 1-minute bars + ohlcv_1m = store.to_ohlcv(timeframe="1m") + assert ohlcv_1m.height >= 1 + + # Test 5-minute bars + ohlcv_5m = store.to_ohlcv(timeframe="5m") + assert ohlcv_5m.height >= 1 + + +def test_capture_get_ohlcv(): + """Test that Capture.get_ohlcv() delegates to store.""" + from dxtrade.capture import Capture + from dxtrade.capture import QuoteStore + + cap = Capture(data_dir=tempfile.gettempdir()) + cap.store = QuoteStore(tempfile.gettempdir()) + cap.store.append( + { + "symbol": "CL", + "bid": 75.5, + "ask": 75.6, + "type": "Quote", + "time": "2026-08-14T12:00:00.000Z", + } + ) + cap.store.flush() + + ohlcv = cap.get_ohlcv(timeframe="1m") + # With 1 quote, OHLCV creates 1 bar (open=high=low=close) + assert ohlcv.height == 1 + assert ohlcv["open"][0] == 75.5 + assert ohlcv["close"][0] == 75.5 + + +def test_parse_timeframe(): + """Test timeframe parsing.""" + from dxtrade.capture import _parse_timeframe + + assert _parse_timeframe("1m") == "1m" + assert _parse_timeframe("5m") == "5m" + assert _parse_timeframe("1h") == "1h" + assert _parse_timeframe("4h") == "4h" + assert _parse_timeframe("1d") == "1d" + assert _parse_timeframe("60") == "60m" # Numeric-only treated as minutes + assert _parse_timeframe("") == "1m" # Empty defaults to 1m + assert _parse_timeframe(None) == "1m" # None defaults to 1m