From 67de5523277394bfadd615bbc005f2bb088ac0cf Mon Sep 17 00:00:00 2001 From: Bob Date: Mon, 8 Dec 2025 17:43:56 -0800 Subject: [PATCH] mass_deploy fixes --- async_mass_deploy.py | 124 ++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 71 deletions(-) diff --git a/async_mass_deploy.py b/async_mass_deploy.py index 10008e1..fd6cdc5 100755 --- a/async_mass_deploy.py +++ b/async_mass_deploy.py @@ -2,12 +2,7 @@ """ ESP32 Async Mass Deployment Tool Combines parallel flashing (via esptool) with async configuration. -Features: -- Semaphore-limited flashing (prevents USB hub crashes) -- Regex-based boot detection (faster/reliable config) -- Parallel verification """ - import asyncio import serial_asyncio import sys @@ -28,7 +23,13 @@ except ImportError: sys.exit(1) # --- Configuration --- -MAX_CONCURRENT_FLASH = 8 # Limit active flashes to prevent USB brownouts +# MAX_CONCURRENT_FLASH is limited to 8 to prevent USB bus brownouts. +# Flashing an ESP32 causes high current draw (spikes >300mA). +# Triggering 30+ flashes simultaneously would draw >10A, overwhelming +# standard powered USB hubs and causing devices to reset or disconnect. +# This semaphore queues the flashing step while allowing other steps (boot/config) +# to run fully parallel. +MAX_CONCURRENT_FLASH = 8 class Colors: GREEN = '\033[92m' @@ -37,7 +38,6 @@ class Colors: BLUE = '\033[94m' RESET = '\033[0m' -# Logger Adapter for context class DeviceLoggerAdapter(logging.LoggerAdapter): def process(self, msg, kwargs): return '[%s] %s' % (self.extra['connid'], msg), kwargs @@ -53,33 +53,26 @@ class DeployWorker: self.build_dir = build_dir self.flash_sem = flash_sem self.log = DeviceLoggerAdapter(logger, {'connid': port}) - - # Regex Patterns self.regex_ready = re.compile(r'Initialization complete|GPS synced|No WiFi config found', re.IGNORECASE) self.regex_got_ip = re.compile(r'got ip:(\d+\.\d+\.\d+\.\d+)', re.IGNORECASE) async def run(self): - """Main deployment workflow""" try: - # 1. FLASHING PHASE (Protected by Semaphore) + # 1. Flash Phase (Semaphore Limited) async with self.flash_sem: if self.args.erase: if not await self._erase_flash(): return False - if not await self._flash_firmware(): return False - # 2. CONFIG PHASE (Serial interaction) - # We assume flash resets device. We open serial immediately to catch boot. - # Note: We wait a tiny bit to let esptool release the port handle + # 2. Config Phase (Fully Parallel) + # Short sleep to let the port release after esptool closes await asyncio.sleep(0.5) if self.args.ssid and self.args.password: if not await self._configure_device(): return False else: self.log.info(f"{Colors.GREEN}Flash Complete (NVS Preserved){Colors.RESET}") - return True - except Exception as e: self.log.error(f"Worker Exception: {e}") return False @@ -87,35 +80,26 @@ class DeployWorker: async def _erase_flash(self): self.log.info("Erasing flash...") cmd = ['esptool.py', '-p', self.port, '-b', '115200', 'erase_flash'] - - proc = await asyncio.create_subprocess_exec( - *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) + proc = await asyncio.create_subprocess_exec(*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) stdout, stderr = await proc.communicate() - if proc.returncode == 0: self.log.info("Erase successful.") return True - else: - self.log.error(f"Erase failed: {stderr.decode()}") - return False + self.log.error(f"Erase failed: {stderr.decode()}") + return False async def _flash_firmware(self): self.log.info("Flashing firmware...") - # Use relative path for flash_args (must be run from build_dir) - cmd = [ - 'esptool.py', '-p', self.port, '-b', str(self.args.baud), - '--before', 'default_reset', '--after', 'hard_reset', - 'write_flash', '@flash_args' - ] + # Note: flash_args contains relative paths, so we must run from build_dir + cmd = ['esptool.py', '-p', self.port, '-b', str(self.args.baud), + '--before', 'default_reset', '--after', 'hard_reset', + 'write_flash', '@flash_args'] proc = await asyncio.create_subprocess_exec( - *cmd, - cwd=self.build_dir, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE + *cmd, cwd=self.build_dir, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) - # Wait with timeout + try: stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=300) except asyncio.TimeoutError: @@ -126,12 +110,11 @@ class DeployWorker: if proc.returncode == 0: self.log.info("Flash successful.") return True - else: - self.log.error(f"Flash failed: {stderr.decode()}") - return False + + self.log.error(f"Flash failed: {stderr.decode()}") + return False async def _configure_device(self): - """Connects via Serial, waits for boot, sends config""" self.log.info("Connecting to console...") try: reader, writer = await serial_asyncio.open_serial_connection(url=self.port, baudrate=115200) @@ -140,11 +123,9 @@ class DeployWorker: return False try: - # A. Wait for Boot self.log.info("Waiting for boot...") booted = False - end_time = time.time() + 8 # 8s boot timeout - + end_time = time.time() + 8 while time.time() < end_time: try: line_b = await asyncio.wait_for(reader.readline(), timeout=0.5) @@ -155,21 +136,13 @@ class DeployWorker: except asyncio.TimeoutError: continue - if not booted: - # Even if we didn't see the specific line, we might still be able to config - self.log.warning("Boot prompt not detected, trying to config anyway...") - - # B. Send Config self.log.info(f"Sending config for {self.target_ip}...") - config_str = ( - f"CFG\nSSID:{self.args.ssid}\nPASS:{self.args.password}\n" - f"IP:{self.target_ip}\nMASK:{self.args.netmask}\nGW:{self.args.gateway}\n" - f"DHCP:0\nEND\n" - ) + config_str = (f"CFG\nSSID:{self.args.ssid}\nPASS:{self.args.password}\n" + f"IP:{self.target_ip}\nMASK:{self.args.netmask}\nGW:{self.args.gateway}\n" + f"DHCP:0\nEND\n") writer.write(config_str.encode('utf-8')) await writer.drain() - # C. Verify self.log.info("Verifying IP...") start_verify = time.time() while time.time() < start_verify + 10: @@ -183,10 +156,8 @@ class DeployWorker: return True except asyncio.TimeoutError: continue - self.log.error("Config sent, but no IP confirmation received.") return False - except Exception as e: self.log.error(f"Config error: {e}") return False @@ -194,29 +165,32 @@ class DeployWorker: writer.close() await writer.wait_closed() -async def main_async(): +def parse_args(): parser = argparse.ArgumentParser(description='Async ESP32 Mass Deployment') parser.add_argument('-d', '--dir', default=os.getcwd(), help='Project dir') parser.add_argument('-s', '--ssid', help='WiFi SSID') - parser.add_argument('-p', '--password', help='WiFi Password') + parser.add_argument('-P', '--password', help='WiFi Password') parser.add_argument('--start-ip', default='192.168.1.51', help='Start IP') parser.add_argument('-b', '--baud', type=int, default=460800, help='Flash baud') parser.add_argument('--erase', action='store_true', help='Full erase first') parser.add_argument('-g', '--gateway', default='192.168.1.1', help='Gateway') parser.add_argument('-m', '--netmask', default='255.255.255.0', help='Netmask') + return parser.parse_args() - args = parser.parse_args() - +async def run_deployment(args): project_dir = Path(args.dir).resolve() build_dir = project_dir / 'build' - # 1. Build Firmware (Sync blocking) + # 1. Build Firmware (Sync/Blocking is fine here as it's a single pre-step) print(f"{Colors.YELLOW}[1/3] Building Firmware...{Colors.RESET}") proc = await asyncio.create_subprocess_exec( - 'idf.py', 'build', cwd=project_dir, - stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE + 'idf.py', 'build', + cwd=project_dir, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE ) _, stderr = await proc.communicate() + if proc.returncode != 0: print(f"{Colors.RED}Build Failed:\n{stderr.decode()}{Colors.RESET}") return @@ -226,21 +200,19 @@ async def main_async(): return print(f"{Colors.GREEN}Build Complete.{Colors.RESET}") - # 2. Detect Devices + # 2. Detect Devices (Sync call to helper) print(f"{Colors.YELLOW}[2/3] Scanning Devices...{Colors.RESET}") devices = detect_esp32.detect_esp32_devices() if not devices: print(f"{Colors.RED}No devices found.{Colors.RESET}") return - # Sort naturally def natural_keys(d): return [int(c) if c.isdigit() else c for c in re.split(r'(\d+)', d.device)] devices.sort(key=natural_keys) # 3. Deploy print(f"{Colors.YELLOW}[3/3] Deploying to {len(devices)} devices...{Colors.RESET}") - print(f"Max Concurrent Flashes: {MAX_CONCURRENT_FLASH}") try: start_ip_obj = ipaddress.IPv4Address(args.start_ip) @@ -248,6 +220,7 @@ async def main_async(): print("Invalid Start IP") return + # Initialize shared semaphore flash_sem = asyncio.Semaphore(MAX_CONCURRENT_FLASH) tasks = [] @@ -256,6 +229,7 @@ async def main_async(): worker = DeployWorker(dev.device, target_ip, args, build_dir, flash_sem) tasks.append(worker.run()) + # Wait for all workers results = await asyncio.gather(*tasks) # 4. Summary @@ -266,11 +240,19 @@ async def main_async(): print(f"Failed: {Colors.RED}{len(devices) - success}{Colors.RESET}") print(f"{Colors.BLUE}{'='*40}{Colors.RESET}") -if __name__ == '__main__': +def main(): + # 1. Parse Arguments (Sync) + args = parse_args() + + # 2. Configure Event Loop (Platform specific) + if os.name == 'nt': + asyncio.set_event_loop(asyncio.ProactorEventLoop()) + + # 3. Run Async Deployment try: - # Windows loop fix if needed - if os.name == 'nt': - asyncio.set_event_loop(asyncio.ProactorEventLoop()) - asyncio.run(main_async()) + asyncio.run(run_deployment(args)) except KeyboardInterrupt: print("\nCancelled.") + +if __name__ == '__main__': + main()