From 1f3803c75ae39ff2dbb83ba10d000a40bc50dea6 Mon Sep 17 00:00:00 2001 From: Robert McMahon Date: Fri, 10 Apr 2026 11:21:26 -0700 Subject: [PATCH] Use orchestrator package layout; docs and tests for remote_nodes - Move src/fiwicontrol to src/orchestrator; imports orchestrator.* - Remove REFACTOR-REVIEW.md; expand Running tests in design doc - README: distribution fiwicontrol, link to test instructions Made-with: Cursor --- README.md | 4 +- docs/REFACTOR-REVIEW.md | 29 ----- docs/remote-nodes-asyncio-design.md | 104 ++++++++++++++---- pyproject.toml | 2 +- src/fiwicontrol/__init__.py | 5 - src/orchestrator/__init__.py | 5 + .../commands/__init__.py | 4 +- .../commands/remote_nodes.py | 3 +- .../power/__init__.py | 2 +- tests/test_package_layout.py | 8 +- tests/test_remote_nodes.py | 6 +- 11 files changed, 105 insertions(+), 67 deletions(-) delete mode 100644 docs/REFACTOR-REVIEW.md delete mode 100644 src/fiwicontrol/__init__.py create mode 100644 src/orchestrator/__init__.py rename src/{fiwicontrol => orchestrator}/commands/__init__.py (74%) rename src/{fiwicontrol => orchestrator}/commands/remote_nodes.py (99%) rename src/{fiwicontrol => orchestrator}/power/__init__.py (61%) diff --git a/README.md b/README.md index 7f6ed63..c51bbdf 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ from orchestrator.commands import ssh_node, Command, CommandManager ## Tests -Commands, example **`pytest`** output, and **`unittest`** entry points for **`remote_nodes`**: **`docs/remote-nodes-asyncio-design.md`** → section **“Running tests”** (top of that file). +Commands, example **`pytest`** output, and **`unittest`** entry points for **`remote_nodes`**: **`docs/remote-nodes-asyncio-design.md`** → section **“Running tests”** (top-level `##` heading near the top of the file). Layout / import smoke (no network): @@ -63,7 +63,7 @@ cd ~/Code/FiWiControl FIWI_REMOTE_IP=192.168.1.39 pytest -q tests/test_remote_nodes.py ``` -## Remote +## Remote (Gitea) ```bash git remote add origin "https://git.umbernetworks.com/rjmcmahon/FiWiControl.git" diff --git a/docs/REFACTOR-REVIEW.md b/docs/REFACTOR-REVIEW.md deleted file mode 100644 index a9e7f5e..0000000 --- a/docs/REFACTOR-REVIEW.md +++ /dev/null @@ -1,29 +0,0 @@ -# Remote nodes module status - -`src/fiwicontrol/commands/remote_nodes.py` is the active asyncio implementation (package **`fiwicontrol.commands`**). - -## Current state - -- Refactor is merged into `remote_nodes.py`; there is no separate `ssh_nodes_refactored.py`. -- Module runs under Python 3.11+ asyncio patterns (`get_running_loop`, async class methods). -- `ssh_node.open_consoles` and `ssh_node.close_consoles` are async class methods. -- `Command` owns scheduling with deadline-based timing and supports: - - finite count (`count=N`) - - never-ending count (`count=Command.COUNT_FOREVER`) -- `CommandManager` can add commands dynamically, self-cleans active commands when done, and keeps abnormal-stop history (exception/timeout). -- Stream logging format includes timestamp (ms), node/ip label, optional run count, command tag, and line text. - -## Tests - -- Active test file: `tests/test_remote_nodes.py` -- Covers: - - single command runs - - dmesg command run - - repeat runs (`uname -r`, `pwd` + `ls`) - - never-ending repeat stop behavior - - `CommandManager` add + self-clean - -## Notes - -- **Passwordless SSH** to `root@` (or your configured user) is required for non-interactive remote runs; see `remote-nodes-asyncio-design.md`. -- Remaining SSH warning output from host config permissions is environment-specific (`/etc/ssh/ssh_config.d/...`) and not a module logic failure. diff --git a/docs/remote-nodes-asyncio-design.md b/docs/remote-nodes-asyncio-design.md index 8f9c1c9..9313a92 100644 --- a/docs/remote-nodes-asyncio-design.md +++ b/docs/remote-nodes-asyncio-design.md @@ -1,16 +1,83 @@ -# Design: Remote nodes — asyncio, streaming, and batched concurrency +# Design: `orchestrator.commands` — asyncio execution (SSH / `ush`) **Status:** Current -**Scope:** `src/fiwicontrol/commands/remote_nodes.py` — asyncio-based SSH/`ush` rig control: **`ssh_node`**, **`ssh_session`**, **`Command`**, **`CommandManager`**, concurrent execution via **`asyncio`**, line-oriented streaming, and logging. + +**Package:** **`orchestrator.commands`** (FiWiControl repository). +**Implementation file:** **`src/orchestrator/commands/remote_nodes.py`**. +The main entry type is still named **`ssh_node`** (historical name: SSH/`ush`-oriented rig connection). + +**Scope:** Asyncio-based SSH/`ush` rig control: **`ssh_node`**, **`ssh_session`**, **`Command`**, **`CommandManager`**, concurrent execution via **`asyncio`**, line-oriented streaming, and logging. **Language:** **Python 3.11+** (stdlib **`asyncio.TaskGroup`**, **`asyncio.timeout`**, and related APIs assumed available). **Remote access:** For **`sshtype="ssh"`**, **passwordless SSH is required**. Commands are executed via **`ssh`** without an interactive TTY, so **key-based authentication** (or any non-interactive auth that does not prompt) must already work for **`root@`** (or your chosen user if you extend argv). Password prompts, interactive MFA, or unlocking passphrase-protected keys in this non-interactive path will cause failures or indefinite waits. --- +## Running tests + +Integration tests for **`remote_nodes.py`** live in **`tests/test_remote_nodes.py`**. They open real **`ssh`** sessions to **`root@$FIWI_REMOTE_IP`** (default **`192.168.1.39`**). **Passwordless SSH must work** for that host. + +**From a terminal** (repository root — e.g. **`~/Code/FiWiControl`**): + +```bash +cd ~/Code/FiWiControl + +# Optional: target a different host (default is 192.168.1.39 if unset) +export FIWI_REMOTE_IP=192.168.1.39 + +# Verbose pytest (recommended) +python -m pytest tests/test_remote_nodes.py -v + +# Quiet summary only +python -m pytest tests/test_remote_nodes.py -q +``` + +**Without pytest**, the file is also a **`unittest`** script: + +```bash +cd ~/Code/FiWiControl +python tests/test_remote_nodes.py --remote-ip 192.168.1.39 +# More asyncio logging: +python tests/test_remote_nodes.py --remote-ip 192.168.1.39 --debug +``` + +**`pytest`** picks up **`pythonpath = ["src"]`** from **`pyproject.toml`**, so you do **not** need **`PYTHONPATH=src`** when you run from the repo root as above. After **`pip install -e .`**, imports resolve the same way. + +**Example successful output** (line order and timings vary; captured with **`python -m pytest tests/test_remote_nodes.py -v`** on Linux): + +```text +============================= test session starts ============================== +platform linux -- Python 3.13.11, pytest-9.0.3, pluggy-1.6.0 -- /usr/bin/python +cachedir: .pytest_cache +rootdir: /home/.../FiWiControl +configfile: pyproject.toml +plugins: anyio-4.8.0 +collecting ... collected 6 items + +tests/test_remote_nodes.py::TestRemoteSingleRunCommands::test_dmesg_tail PASSED [ 16%] +tests/test_remote_nodes.py::TestRemoteSingleRunCommands::test_ls_home_directory PASSED [ 33%] +tests/test_remote_nodes.py::TestRemoteRepeatingCommands::test_command_manager_add_and_self_clean PASSED [ 50%] +tests/test_remote_nodes.py::TestRemoteRepeatingCommands::test_repeat_01_uname_r_every_1s_10_times PASSED [ 66%] +tests/test_remote_nodes.py::TestRemoteRepeatingCommands::test_repeat_02_pwd_and_ls_every_1s_5_times PASSED [ 83%] +tests/test_remote_nodes.py::TestRemoteRepeatingCommands::test_repeat_03_forever_can_be_stopped PASSED [100%] + +============================== 6 passed in 16.47s ============================== +``` + +If SSH is not configured, you will see failures (connection refused, auth errors, or timeouts) instead of **`PASSED`**. + +**Package import smoke** (no network; optional): + +```bash +cd ~/Code/FiWiControl +python -m pytest tests/test_package_layout.py -q +``` + +--- + ## How to use (examples) -Implementation lives in **`src/fiwicontrol/commands/remote_nodes.py`**. All remote I/O is **async** — call from a coroutine under **`asyncio.run()`** or your application event loop. +Implementation lives in **`src/orchestrator/commands/remote_nodes.py`**. All remote I/O is **async** — call from a coroutine under **`asyncio.run()`** or your application event loop. ### Prerequisites (remote targets) @@ -18,16 +85,26 @@ Implementation lives in **`src/fiwicontrol/commands/remote_nodes.py`**. All remo 2. Network reachability and correct **`ipaddr`** (or hostname) on the node. 3. Optional: **`BatchMode=yes`** in **`ssh_config`** for fail-fast behavior when keys are missing (recommended for automation). -### Import and layout +### Imports + +Preferred (re-exported from **`orchestrator.commands`**): ```python import asyncio -from fiwicontrol.commands.remote_nodes import Command, CommandManager, ssh_node +from orchestrator.commands import Command, CommandManager, ssh_node ``` -Install the package (editable: `pip install -e .` from the **FiWiControl** repo root), or run tests with **`pytest`** using **`pythonpath = ["src"]`** in **`pyproject.toml`**. +Equivalent submodule import (same symbols): -### Create a node (`ssh_node`) +```python +from orchestrator.commands.remote_nodes import Command, CommandManager, ssh_node +``` + +Install the package (editable: **`pip install -e .`** from the **FiWiControl** repo root; distribution name **`fiwicontrol`**). For **`pytest`** without installing, **`pythonpath = ["src"]`** is set in **`pyproject.toml`**. + +### Create an `ssh_node` (rig / execution target) + +**`ssh_node`** is a **class** in **`orchestrator.commands`**: one instance represents **one** reachable rig (OpenSSH or **`ush`**). It is not the package name; the package is **`orchestrator.commands`**. **SSH (default)** — targets **`root@ipaddr`** with ControlMaster socket under **`/tmp/controlmasters_`** when **`ssh_controlmaster=True`** (default for **`rexec`** / **`Command`** sessions). @@ -159,18 +236,7 @@ await ssh_node.open_consoles(silent_mode=False) await ssh_node.close_consoles() ``` -### Automated tests - -**`tests/test_remote_nodes.py`** exercises remote shell access when **`FIWI_REMOTE_IP`** is reachable. Run, overriding the default IP: - -```bash -# From repository root (FiWiControl): -cd /path/to/FiWiControl -FIWI_REMOTE_IP=192.168.1.39 python -m pytest tests/test_remote_nodes.py -v - -# Or run the unittest module directly: -python tests/test_remote_nodes.py --remote-ip 192.168.1.39 -``` +**Tests:** see **[Running tests](#running-tests)** at the top of this document. --- diff --git a/pyproject.toml b/pyproject.toml index 9e1ee75..891d8bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "fiwicontrol" version = "0.1.0" -description = "FiWiControl: command execution (SSH/ush) and power rig control in one package." +description = "FiWiControl repo: orchestrator.commands (SSH/ush) and orchestrator.power in one installable package." readme = "README.md" requires-python = ">=3.11" license = { file = "LICENSE" } diff --git a/src/fiwicontrol/__init__.py b/src/fiwicontrol/__init__.py deleted file mode 100644 index c25b8fc..0000000 --- a/src/fiwicontrol/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""FiWiControl: command execution and power rig helpers (single installable package).""" - -__all__ = ["__version__"] - -__version__ = "0.1.0" diff --git a/src/orchestrator/__init__.py b/src/orchestrator/__init__.py new file mode 100644 index 0000000..11736b4 --- /dev/null +++ b/src/orchestrator/__init__.py @@ -0,0 +1,5 @@ +"""Orchestrator package: command execution and power rig helpers (FiWiControl repository).""" + +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/src/fiwicontrol/commands/__init__.py b/src/orchestrator/commands/__init__.py similarity index 74% rename from src/fiwicontrol/commands/__init__.py rename to src/orchestrator/commands/__init__.py index 13e3755..cd3c7c5 100644 --- a/src/fiwicontrol/commands/__init__.py +++ b/src/orchestrator/commands/__init__.py @@ -1,10 +1,10 @@ """ Command execution over SSH / ``ush``: sessions, scheduling, timeouts. -Must not import from ``fiwicontrol.power``. +Must not import from ``orchestrator.power``. """ -from fiwicontrol.commands.remote_nodes import ( +from orchestrator.commands.remote_nodes import ( Command, CommandManager, sleep_async, diff --git a/src/fiwicontrol/commands/remote_nodes.py b/src/orchestrator/commands/remote_nodes.py similarity index 99% rename from src/fiwicontrol/commands/remote_nodes.py rename to src/orchestrator/commands/remote_nodes.py index 3117799..aad1bee 100644 --- a/src/fiwicontrol/commands/remote_nodes.py +++ b/src/orchestrator/commands/remote_nodes.py @@ -3,6 +3,7 @@ # Licensed under the Apache License, Version 2.0; see LICENSE. # # Remote command helpers (SSH / ush) for test rig control. +# Package: orchestrator.commands (FiWiControl). Public imports: from orchestrator.commands import ssh_node, ... # # Async-first implementation for ssh_node. # @@ -11,7 +12,7 @@ # - Repeating: cmd = Command(node, cmd="...", interval=1.0, count=10); await cmd.start(); await cmd.task # - Many: CommandManager().add_command(...) / stop_all() / list_active() # -# Remote targets (sshtype "ssh"): passwordless SSH is required — see docs/remote-nodes-asyncio-design.md. +# Remote targets (sshtype "ssh"): passwordless SSH is required — see docs/remote-nodes-asyncio-design.md (## Running tests). from __future__ import annotations diff --git a/src/fiwicontrol/power/__init__.py b/src/orchestrator/power/__init__.py similarity index 61% rename from src/fiwicontrol/power/__init__.py rename to src/orchestrator/power/__init__.py index 37ca50f..aee8b7b 100644 --- a/src/fiwicontrol/power/__init__.py +++ b/src/orchestrator/power/__init__.py @@ -1,7 +1,7 @@ """ Power switching, monitoring, and discovery (Acroname, Monsoon, …). -May import ``fiwicontrol.commands``; the reverse is forbidden. +May import ``orchestrator.commands``; the reverse is forbidden. """ __all__: list[str] = [] diff --git a/tests/test_package_layout.py b/tests/test_package_layout.py index b6a321b..5a1ccea 100644 --- a/tests/test_package_layout.py +++ b/tests/test_package_layout.py @@ -1,6 +1,6 @@ def test_import_subpackages() -> None: - import fiwicontrol - import fiwicontrol.commands - import fiwicontrol.power + import orchestrator + import orchestrator.commands + import orchestrator.power - assert fiwicontrol.__version__ + assert orchestrator.__version__ diff --git a/tests/test_remote_nodes.py b/tests/test_remote_nodes.py index d4f802c..1047d59 100644 --- a/tests/test_remote_nodes.py +++ b/tests/test_remote_nodes.py @@ -5,7 +5,7 @@ import os import sys import unittest -from fiwicontrol.commands.remote_nodes import Command, CommandManager, ssh_node +from orchestrator.commands.remote_nodes import Command, CommandManager, ssh_node class _RemoteNodeMixin: @@ -84,7 +84,7 @@ class TestRemoteRepeatingCommands(_RemoteNodeMixin, unittest.IsolatedAsyncioTest if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Integration tests for fiwicontrol.commands.remote_nodes (passwordless SSH required)." + description="Integration tests for orchestrator.commands.remote_nodes (passwordless SSH required)." ) parser.add_argument( "--remote-ip", @@ -100,5 +100,5 @@ if __name__ == "__main__": else: logging.basicConfig(level=logging.WARNING, format="%(message)s") logging.getLogger("asyncio").setLevel(logging.WARNING) - logging.getLogger("fiwicontrol.commands.remote_nodes").setLevel(logging.INFO) + logging.getLogger("orchestrator.commands.remote_nodes").setLevel(logging.INFO) unittest.main(argv=[sys.argv[0], *remaining])