mass deploy

This commit is contained in:
Bob 2025-11-15 12:07:35 -08:00
parent 60c1d49d3e
commit f2abcf450b
2 changed files with 129 additions and 100 deletions

View File

@ -89,7 +89,7 @@ static void led_task(void *arg)
case LED_STATE_CONNECTED: case LED_STATE_CONNECTED:
// Blue solid - connected successfully // Blue solid - connected successfully
set_led_color(0, 0, 255); // Blue set_led_color(0, 255, 0); // Green
vTaskDelay(pdMS_TO_TICKS(1000)); vTaskDelay(pdMS_TO_TICKS(1000));
break; break;

View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
ESP32 Mass Deployment Tool ESP32 Mass Deployment Tool
Parallel or sequential flashing and WiFi configuration for multiple ESP32 devices Parallel flashing and WiFi configuration for multiple ESP32 devices
""" """
import os import os
@ -25,7 +25,7 @@ class DeviceDeployer:
netmask="255.255.255.0", gateway="192.168.1.1", netmask="255.255.255.0", gateway="192.168.1.1",
baud_rate=460800, max_retries=2, verify_ping=True, baud_rate=460800, max_retries=2, verify_ping=True,
num_devices=None, verbose=False, parallel=True): num_devices=None, verbose=False, parallel=True):
self.project_dir = Path(project_dir) self.project_dir = Path(project_dir)
self.ssid = ssid self.ssid = ssid
self.password = password self.password = password
@ -38,15 +38,16 @@ class DeviceDeployer:
self.num_devices = num_devices self.num_devices = num_devices
self.verbose = verbose self.verbose = verbose
self.parallel = parallel self.parallel = parallel
# Parse IP address # Parse IP address
ip_parts = start_ip.split('.') ip_parts = start_ip.split('.')
self.ip_base = '.'.join(ip_parts[:3]) self.ip_base = '.'.join(ip_parts[:3])
self.ip_start = int(ip_parts[3]) self.ip_start = int(ip_parts[3])
self.devices = [] self.devices = []
self.results = {} self.results = {}
self.log_dir = Path('/tmp')
def print_banner(self): def print_banner(self):
"""Print deployment configuration""" """Print deployment configuration"""
print() print()
@ -66,12 +67,12 @@ class DeviceDeployer:
if self.num_devices: if self.num_devices:
print(f"Max Devices: {self.num_devices}") print(f"Max Devices: {self.num_devices}")
print(f"{Colors.BLUE}{'='*70}{Colors.NC}") print(f"{Colors.BLUE}{'='*70}{Colors.NC}")
def build_firmware(self): def build_firmware(self):
"""Build the firmware using idf.py""" """Build the firmware using idf.py"""
print() print()
print(f"{Colors.YELLOW}[1/4] Building firmware...{Colors.NC}") print(f"{Colors.YELLOW}[1/4] Building firmware...{Colors.NC}")
try: try:
result = subprocess.run( result = subprocess.run(
['idf.py', 'build'], ['idf.py', 'build'],
@ -86,64 +87,68 @@ class DeviceDeployer:
if self.verbose: if self.verbose:
print(e.stderr.decode() if e.stderr else "") print(e.stderr.decode() if e.stderr else "")
return False return False
def detect_devices(self): def detect_devices(self):
"""Detect connected ESP32 devices""" """Detect connected ESP32 devices"""
print() print()
print(f"{Colors.YELLOW}[2/4] Detecting ESP32 devices...{Colors.NC}") print(f"{Colors.YELLOW}[2/4] Detecting ESP32 devices...{Colors.NC}")
# Find all USB serial devices # Find all USB serial devices
self.devices = sorted(glob.glob('/dev/ttyUSB*') + glob.glob('/dev/ttyACM*')) self.devices = sorted(glob.glob('/dev/ttyUSB*') + glob.glob('/dev/ttyACM*'))
if not self.devices: if not self.devices:
print(f"{Colors.RED}ERROR: No devices found!{Colors.NC}") print(f"{Colors.RED}ERROR: No devices found!{Colors.NC}")
print("Connect ESP32 devices via USB and try again.") print("Connect ESP32 devices via USB and try again.")
return False return False
# Limit to num_devices if specified # Limit to num_devices if specified
if self.num_devices and len(self.devices) > self.num_devices: if self.num_devices and len(self.devices) > self.num_devices:
print(f"Limiting to first {self.num_devices} devices") print(f"Limiting to first {self.num_devices} devices")
self.devices = self.devices[:self.num_devices] self.devices = self.devices[:self.num_devices]
print(f"{Colors.GREEN}Found {len(self.devices)} device(s):{Colors.NC}") print(f"{Colors.GREEN}Found {len(self.devices)} device(s):{Colors.NC}")
for i, device in enumerate(self.devices): for i, device in enumerate(self.devices):
ip = self.get_ip_for_index(i) ip = self.get_ip_for_index(i)
print(f" [{i:2d}] {device:14s}{ip}") print(f" [{i:2d}] {device:14s}{ip}")
return True return True
def get_ip_for_index(self, index): def get_ip_for_index(self, index):
"""Calculate IP address for device index""" """Calculate IP address for device index"""
return f"{self.ip_base}.{self.ip_start + index}" return f"{self.ip_base}.{self.ip_start + index}"
def flash_and_configure(self, index, device): def flash_and_configure(self, index, device):
"""Flash and configure a single device""" """Flash and configure a single device"""
ip_addr = self.get_ip_for_index(index) ip_addr = self.get_ip_for_index(index)
log_file = self.log_dir / f"esp32_deploy_{index}.log"
log_lines = [] log_lines = []
def log(msg): def log(msg):
log_lines.append(msg) log_lines.append(msg)
if self.verbose or not self.parallel: if self.verbose or not self.parallel:
print(f"[{index}] {msg}") print(f"[{index}] {msg}")
for attempt in range(1, self.max_retries + 1): for attempt in range(1, self.max_retries + 1):
log(f"=== Device {index}: {device} (Attempt {attempt}/{self.max_retries}) ===") log(f"=== Device {index}: {device} (Attempt {attempt}/{self.max_retries}) ===")
log(f"Target IP: {ip_addr}") log(f"Target IP: {ip_addr}")
# Flash firmware # Flash firmware
log("Flashing...") log("Flashing...")
try: try:
subprocess.run( result = subprocess.run(
['idf.py', '-p', device, '-b', str(self.baud_rate), 'flash'], ['idf.py', '-p', device, '-b', str(self.baud_rate), 'flash'],
cwd=self.project_dir, cwd=self.project_dir,
check=True, check=True,
capture_output=not (self.verbose or not self.parallel), capture_output=True,
timeout=300 # 5 minute timeout timeout=300 # 5 minute timeout
) )
log("✓ Flash successful") log("✓ Flash successful")
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log(f"✗ Flash failed on attempt {attempt}") log(f"✗ Flash failed on attempt {attempt}")
if attempt == self.max_retries: if attempt == self.max_retries:
# Write log file
with open(log_file, 'w') as f:
f.write('\n'.join(log_lines))
return { return {
'index': index, 'index': index,
'device': device, 'device': device,
@ -156,6 +161,8 @@ class DeviceDeployer:
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
log(f"✗ Flash timeout on attempt {attempt}") log(f"✗ Flash timeout on attempt {attempt}")
if attempt == self.max_retries: if attempt == self.max_retries:
with open(log_file, 'w') as f:
f.write('\n'.join(log_lines))
return { return {
'index': index, 'index': index,
'device': device, 'device': device,
@ -164,11 +171,11 @@ class DeviceDeployer:
'log': log_lines 'log': log_lines
} }
continue continue
# Wait for device to boot # Wait for device to boot
log("Waiting for boot...") log("Waiting for boot...")
time.sleep(3) time.sleep(3)
# Configure WiFi # Configure WiFi
log("Configuring WiFi...") log("Configuring WiFi...")
try: try:
@ -182,18 +189,18 @@ class DeviceDeployer:
f"DHCP:0\n" f"DHCP:0\n"
f"END\n" f"END\n"
) )
with open(device, 'w') as f: with open(device, 'w') as f:
f.write(config) f.write(config)
log("✓ Config sent") log("✓ Config sent")
except Exception as e: except Exception as e:
log(f"✗ Config error: {e}") log(f"✗ Config error: {e}")
# Wait for network to initialize # Wait for network to initialize
log("Waiting for network...") log("Waiting for network...")
time.sleep(5) time.sleep(5)
# Verify connectivity # Verify connectivity
if self.verify_ping: if self.verify_ping:
log("Verifying connectivity...") log("Verifying connectivity...")
@ -203,9 +210,12 @@ class DeviceDeployer:
capture_output=True, capture_output=True,
timeout=10 timeout=10
) )
if result.returncode == 0: if result.returncode == 0:
log("✓ Ping successful") log("✓ Ping successful")
# Write log file
with open(log_file, 'w') as f:
f.write('\n'.join(log_lines))
return { return {
'index': index, 'index': index,
'device': device, 'device': device,
@ -216,6 +226,8 @@ class DeviceDeployer:
else: else:
log(f"✗ Ping failed on attempt {attempt}") log(f"✗ Ping failed on attempt {attempt}")
if attempt == self.max_retries: if attempt == self.max_retries:
with open(log_file, 'w') as f:
f.write('\n'.join(log_lines))
return { return {
'index': index, 'index': index,
'device': device, 'device': device,
@ -226,6 +238,9 @@ class DeviceDeployer:
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
log(f"✗ Ping timeout on attempt {attempt}") log(f"✗ Ping timeout on attempt {attempt}")
else: else:
# Write log file
with open(log_file, 'w') as f:
f.write('\n'.join(log_lines))
return { return {
'index': index, 'index': index,
'device': device, 'device': device,
@ -233,10 +248,12 @@ class DeviceDeployer:
'status': 'SUCCESS', 'status': 'SUCCESS',
'log': log_lines 'log': log_lines
} }
time.sleep(2) time.sleep(2)
# If we get here, all retries failed # If we get here, all retries failed
with open(log_file, 'w') as f:
f.write('\n'.join(log_lines))
return { return {
'index': index, 'index': index,
'device': device, 'device': device,
@ -244,46 +261,54 @@ class DeviceDeployer:
'status': 'FAILED', 'status': 'FAILED',
'log': log_lines 'log': log_lines
} }
def deploy_all_parallel(self): def deploy_all_parallel(self):
"""Deploy to all devices in parallel""" """Deploy to all devices in parallel"""
print() print()
print(f"{Colors.YELLOW}[3/4] Flashing and configuring (parallel)...{Colors.NC}") print(f"{Colors.YELLOW}[3/4] Flashing and configuring (parallel)...{Colors.NC}")
print() print()
# Clean old log files
for f in self.log_dir.glob('esp32_deploy_*.log'):
f.unlink()
# Use ThreadPoolExecutor for parallel execution # Use ThreadPoolExecutor for parallel execution
max_workers = min(32, len(self.devices)) max_workers = min(32, len(self.devices))
with ThreadPoolExecutor(max_workers=max_workers) as executor: with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit all jobs # Submit all jobs
futures = { futures = {
executor.submit(self.flash_and_configure, i, device): (i, device) executor.submit(self.flash_and_configure, i, device): (i, device)
for i, device in enumerate(self.devices) for i, device in enumerate(self.devices)
} }
# Collect results as they complete # Collect results as they complete
for future in as_completed(futures): for future in as_completed(futures):
result = future.result() result = future.result()
self.results[result['index']] = result self.results[result['index']] = result
# Print immediate status # Print immediate status
self.print_device_status(result) self.print_device_status(result)
def deploy_all_sequential(self): def deploy_all_sequential(self):
"""Deploy to devices one at a time (sequential)""" """Deploy to devices one at a time (sequential)"""
print() print()
print(f"{Colors.YELLOW}[3/4] Flashing and configuring (sequential)...{Colors.NC}") print(f"{Colors.YELLOW}[3/4] Flashing and configuring (sequential)...{Colors.NC}")
print() print()
# Clean old log files
for f in self.log_dir.glob('esp32_deploy_*.log'):
f.unlink()
for i, device in enumerate(self.devices): for i, device in enumerate(self.devices):
print(f"\n{Colors.BLUE}--- Device {i+1}/{len(self.devices)} ---{Colors.NC}") print(f"\n{Colors.BLUE}--- Device {i+1}/{len(self.devices)} ---{Colors.NC}")
result = self.flash_and_configure(i, device) result = self.flash_and_configure(i, device)
self.results[result['index']] = result self.results[result['index']] = result
# Print status after each device # Print status after each device
self.print_device_status(result) self.print_device_status(result)
print() print()
def print_device_status(self, result): def print_device_status(self, result):
"""Print status for a single device""" """Print status for a single device"""
status_color = { status_color = {
@ -292,35 +317,35 @@ class DeviceDeployer:
'FAILED': Colors.RED, 'FAILED': Colors.RED,
'TIMEOUT': Colors.RED 'TIMEOUT': Colors.RED
}.get(result['status'], Colors.RED) }.get(result['status'], Colors.RED)
status_text = { status_text = {
'SUCCESS': 'OK', 'SUCCESS': 'OK',
'NO_PING': 'FLASHED, NO PING', 'NO_PING': 'FLASHED, NO PING',
'FAILED': 'FAILED', 'FAILED': 'FAILED',
'TIMEOUT': 'TIMEOUT' 'TIMEOUT': 'TIMEOUT'
}.get(result['status'], 'ERROR') }.get(result['status'], 'ERROR')
print(f"{status_color}[Device {result['index']:2d}] {result['device']:14s}" print(f"{status_color}[Device {result['index']:2d}] {result['device']:14s}"
f"{result['ip']:15s} [{status_text}]{Colors.NC}") f"{result['ip']:15s} [{status_text}]{Colors.NC}")
def deploy_all(self): def deploy_all(self):
"""Deploy to all devices (parallel or sequential)""" """Deploy to all devices (parallel or sequential)"""
if self.parallel: if self.parallel:
self.deploy_all_parallel() self.deploy_all_parallel()
else: else:
self.deploy_all_sequential() self.deploy_all_sequential()
def print_summary(self): def print_summary(self):
"""Print deployment summary""" """Print deployment summary"""
print() print()
print(f"{Colors.YELLOW}[4/4] Deployment Summary{Colors.NC}") print(f"{Colors.YELLOW}[4/4] Deployment Summary{Colors.NC}")
print(f"{Colors.BLUE}{'='*70}{Colors.NC}") print(f"{Colors.BLUE}{'='*70}{Colors.NC}")
# Count statuses # Count statuses
success_count = sum(1 for r in self.results.values() if r['status'] == 'SUCCESS') success_count = sum(1 for r in self.results.values() if r['status'] == 'SUCCESS')
no_ping_count = sum(1 for r in self.results.values() if r['status'] == 'NO_PING') no_ping_count = sum(1 for r in self.results.values() if r['status'] == 'NO_PING')
failed_count = sum(1 for r in self.results.values() if r['status'] in ['FAILED', 'TIMEOUT']) failed_count = sum(1 for r in self.results.values() if r['status'] in ['FAILED', 'TIMEOUT'])
# Print all devices # Print all devices
for i in range(len(self.devices)): for i in range(len(self.devices)):
if i in self.results: if i in self.results:
@ -331,41 +356,42 @@ class DeviceDeployer:
'FAILED': f"{Colors.RED}{Colors.NC}", 'FAILED': f"{Colors.RED}{Colors.NC}",
'TIMEOUT': f"{Colors.RED}{Colors.NC}" 'TIMEOUT': f"{Colors.RED}{Colors.NC}"
}.get(result['status'], f"{Colors.RED}?{Colors.NC}") }.get(result['status'], f"{Colors.RED}?{Colors.NC}")
status_msg = { status_msg = {
'NO_PING': " (no ping response)", 'NO_PING': " (no ping response)",
'FAILED': " (failed)", 'FAILED': " (failed)",
'TIMEOUT': " (timeout)" 'TIMEOUT': " (timeout)"
}.get(result['status'], "") }.get(result['status'], "")
print(f"{status_icon} {result['device']:14s}{result['ip']}{status_msg}") print(f"{status_icon} {result['device']:14s}{result['ip']}{status_msg}")
print(f"{Colors.BLUE}{'='*70}{Colors.NC}") print(f"{Colors.BLUE}{'='*70}{Colors.NC}")
print(f"Total: {len(self.devices)} devices") print(f"Total: {len(self.devices)} devices")
print(f"{Colors.GREEN}Success: {success_count}{Colors.NC}") print(f"{Colors.GREEN}Success: {success_count}{Colors.NC}")
if no_ping_count > 0: if no_ping_count > 0:
print(f"{Colors.YELLOW}Warning: {no_ping_count} (flashed but no ping){Colors.NC}") print(f"{Colors.YELLOW}Warning: {no_ping_count} (flashed but no ping){Colors.NC}")
if failed_count > 0: if failed_count > 0:
print(f"{Colors.RED}Failed: {failed_count}{Colors.NC}") print(f"{Colors.RED}Failed: {failed_count}{Colors.NC}")
print(f"{Colors.BLUE}{'='*70}{Colors.NC}") print(f"{Colors.BLUE}{'='*70}{Colors.NC}")
# Print log location
print()
print(f"Logs: /tmp/esp32_deploy_*.log")
# Print test commands # Print test commands
print() print()
print("Test commands:") print("Test commands:")
print(f" # Test first device")
print(f" iperf -c {self.get_ip_for_index(0)}")
print()
print(f" # Ping all devices") print(f" # Ping all devices")
ip_range = f"{self.ip_start}..{self.ip_start + len(self.devices) - 1}" ip_range = f"{self.ip_start}..{self.ip_start + len(self.devices) - 1}"
print(f" for i in {{{ip_range}}}; do ping -c 1 {self.ip_base}.$i & done; wait") print(f" for i in {{{ip_range}}}; do ping -c 1 {self.ip_base}.$i & done; wait")
print() print()
print(f" # Test first device with iperf")
print(f" iperf -c {self.get_ip_for_index(0)}")
print()
print(f" # Check all device status")
print(f" ./check_device_status.py --reset")
print()
return failed_count return failed_count
def main(): def main():
@ -374,72 +400,75 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=""" epilog="""
Examples: Examples:
# Deploy to first 15 devices (parallel, default) # Deploy to first 30 devices (default)
%(prog)s -s ClubHouse2G -p mypassword %(prog)s -s ClubHouse2G -p mypassword
# Deploy sequentially (easier debugging) # Deploy sequentially (easier debugging)
%(prog)s -s ClubHouse2G -p mypassword --sequential %(prog)s -s ClubHouse2G -p mypassword --sequential
# Deploy to 31 devices in parallel # Deploy to all connected devices
%(prog)s -s ClubHouse2G -p mypassword -n 31 %(prog)s -s ClubHouse2G -p mypassword -n 0
# Custom IP range # Custom IP range
%(prog)s -s ClubHouse2G -p mypassword --start-ip 10.0.0.100 %(prog)s -s ClubHouse2G -p mypassword --start-ip 10.0.0.100
# From environment variables # From environment variables
export WIFI_SSID="MyNetwork" export SSID="MyNetwork"
export WIFI_PASSWORD="secret123" export PASSWORD="secret123"
export START_IP="192.168.1.51"
%(prog)s %(prog)s
# Verbose sequential mode (see everything) # Verbose sequential mode (see everything)
%(prog)s -s ClubHouse2G -p mypassword --sequential -v %(prog)s -s ClubHouse2G -p mypassword --sequential -v
# Skip ping verification (faster)
%(prog)s -s ClubHouse2G -p mypassword --no-verify
""" """
) )
parser.add_argument('-d', '--dir', default=os.getcwd(), parser.add_argument('-d', '--dir', default=os.getcwd(),
help='ESP-IDF project directory (default: current dir)') help='ESP-IDF project directory (default: current dir)')
parser.add_argument('-s', '--ssid', parser.add_argument('-s', '--ssid',
default=os.environ.get('WIFI_SSID', 'ClubHouse2G'), default=os.environ.get('SSID', 'ClubHouse2G'),
help='WiFi SSID (default: ClubHouse2G or $WIFI_SSID)') help='WiFi SSID (default: ClubHouse2G or $SSID)')
parser.add_argument('-p', '--password', parser.add_argument('-p', '--password',
default=os.environ.get('WIFI_PASSWORD', ''), default=os.environ.get('PASSWORD', ''),
help='WiFi password (default: $WIFI_PASSWORD)') help='WiFi password (default: $PASSWORD)')
parser.add_argument('--start-ip', default='192.168.1.51', parser.add_argument('--start-ip',
help='Starting IP address (default: 192.168.1.51)') default=os.environ.get('START_IP', '192.168.1.51'),
parser.add_argument('-n', '--num-devices', type=int, default=15, help='Starting IP address (default: 192.168.1.51 or $START_IP)')
help='Number of devices to deploy (default: 15, use 0 for all)') parser.add_argument('-n', '--num-devices', type=int, default=30,
parser.add_argument('-g', '--gateway', default='192.168.1.1', help='Number of devices to deploy (default: 30, use 0 for all)')
help='Gateway IP (default: 192.168.1.1)') parser.add_argument('-g', '--gateway',
parser.add_argument('-m', '--netmask', default='255.255.255.0', default=os.environ.get('GATEWAY', '192.168.1.1'),
help='Network mask (default: 255.255.255.0)') help='Gateway IP (default: 192.168.1.1 or $GATEWAY)')
parser.add_argument('-b', '--baud', type=int, default=460800, parser.add_argument('-m', '--netmask',
help='Baud rate for flashing (default: 460800)') default=os.environ.get('NETMASK', '255.255.255.0'),
parser.add_argument('-r', '--retries', type=int, default=2, help='Network mask (default: 255.255.255.0 or $NETMASK)')
help='Max retries per device (default: 2)') parser.add_argument('-b', '--baud', type=int,
default=int(os.environ.get('BAUD_RATE', 460800)),
help='Baud rate for flashing (default: 460800 or $BAUD_RATE)')
parser.add_argument('-r', '--retries', type=int,
default=int(os.environ.get('MAX_RETRIES', 2)),
help='Max retries per device (default: 2 or $MAX_RETRIES)')
parser.add_argument('--no-verify', action='store_true', parser.add_argument('--no-verify', action='store_true',
help='Skip ping verification') help='Skip ping verification')
parser.add_argument('--sequential', action='store_true', parser.add_argument('--sequential', action='store_true',
help='Flash devices sequentially instead of parallel (easier debugging)') help='Flash devices sequentially instead of parallel')
parser.add_argument('-v', '--verbose', action='store_true', parser.add_argument('-v', '--verbose', action='store_true',
help='Verbose output') help='Verbose output')
args = parser.parse_args() args = parser.parse_args()
# Validate password # Validate password
if not args.password: if not args.password:
print(f"{Colors.RED}ERROR: WiFi password not set!{Colors.NC}") print(f"{Colors.RED}ERROR: WiFi password not set!{Colors.NC}")
print() print()
print("Provide password via:") print("Provide password via:")
print(" 1. Command line: -p 'your_password'") print(" 1. Command line: -p 'your_password'")
print(" 2. Environment: export WIFI_PASSWORD='your_password'") print(" 2. Environment: export PASSWORD='your_password'")
print() print()
print("Example:") print("Example:")
print(f" {sys.argv[0]} -s MyWiFi -p mypassword") print(f" {sys.argv[0]} -s MyWiFi -p mypassword")
sys.exit(1) sys.exit(1)
# Create deployer # Create deployer
deployer = DeviceDeployer( deployer = DeviceDeployer(
project_dir=args.dir, project_dir=args.dir,
@ -455,19 +484,19 @@ Examples:
verbose=args.verbose, verbose=args.verbose,
parallel=not args.sequential parallel=not args.sequential
) )
# Run deployment # Run deployment
deployer.print_banner() deployer.print_banner()
if not deployer.build_firmware(): if not deployer.build_firmware():
sys.exit(1) sys.exit(1)
if not deployer.detect_devices(): if not deployer.detect_devices():
sys.exit(1) sys.exit(1)
deployer.deploy_all() deployer.deploy_all()
failed_count = deployer.print_summary() failed_count = deployer.print_summary()
sys.exit(failed_count) sys.exit(failed_count)
if __name__ == '__main__': if __name__ == '__main__':