#!/usr/bin/env python3 import os import re import sys import time import datetime import subprocess import json # PTP_CONF = "/opt/ptp_conf/ptp4l_eth_line_top.conf" PTP_REMOTE_CONFIG_PATH = "/opt/ptp_conf/" GM_FILE = "ptp4l_grandmaster.conf" SV_FILE = "ptp4l_slave.conf" MA_FILE = "ptp4l_master.conf" BC_FILE = "ptp4l_boundary.conf" PTP4L_INSTANCE = "ptp4l" # PTP4L_INSTANCE = "/home/apu/wifi-ptp/ptp/ptp4l" GRANDMASTER_LOG = "/tmp/ptp4l_grandmaster.log" SLAVE_LOG = "/tmp/ptp4l_slave.log" MASTER_LOG = "/tmp/ptp4l_master.log" BOUNDARY_LOG = "/tmp/ptp4l_boundary.log" PHC2SYS_LOG = "/tmp/phc2sys.log" PTP_CONFIG_DIR = "ptp_config/eth" def load_ptp_config(filepath): with open(filepath, "r") as f: data = json.load(f) lines = [] for section, options in data.items(): lines.append(f"[{section}]") for key, value in options.items(): lines.append(f"{key} {value}") return "\n".join(lines) def usage(): print("Usage: run_ptp_experiment.py ") print("Example: run_ptp_experiment.py 60 0 1 2 3 4") sys.exit(1) def get_hostname(node_id): return f"apu{str(node_id).zfill(2)}" def write_ptp_config_to_node(hostname, role, conf_str): if role == "grandmaster": file_name = GM_FILE elif role == "slave": file_name = SV_FILE elif role == "master": file_name = MA_FILE elif role == "boundary": file_name = BC_FILE else: raise ValueError("Invalid role. Must be 'grandmaster' or 'slave'.") try: subprocess.run( ["ssh", hostname, f"echo '{conf_str}' | sudo tee '{PTP_REMOTE_CONFIG_PATH}/{file_name}' > /dev/null"], shell=False, check=True ) print(f"[{hostname}] Wrote ptp4l config.") except subprocess.CalledProcessError as e: print(f"ERROR: Failed to write ptp4l.conf to {hostname}: {e}") def generate_output_filename(nodes, duration): timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") node_list = "-".join(str(n) for n in nodes) return f"{timestamp}_ptp4l_{duration}s_nodes({node_list}).log" def delete_logs_on_node(hostname): log_files = [GRANDMASTER_LOG, SLAVE_LOG, MASTER_LOG, PHC2SYS_LOG, BOUNDARY_LOG] delete_cmd = "sudo rm -f " + " ".join(log_files) print(f"[{hostname}] Running: {delete_cmd}") try: result = subprocess.run( ["ssh", hostname, delete_cmd], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) print(f"[{hostname}] Log files deleted.") except subprocess.CalledProcessError as e: print(f"[{hostname}] Failed to delete log files.") print(f"STDERR:\n{e.stderr}") def start_ptp_on_node_old(node_id, nodes): # Old version: using ptp4l boundary clock directly hostname = get_hostname(node_id) min_node = min(nodes) max_node = max(nodes) if node_id == min_node: config = load_ptp_config(os.path.join(PTP_CONFIG_DIR, "ptp4l_grandmaster.json")) write_ptp_config_to_node(hostname, "grandmaster",config) cmd = f"ptp4l -i eth2 -m -H -f {PTP_REMOTE_CONFIG_PATH}{GM_FILE} > {GRANDMASTER_LOG} 2>&1 &" # Grandmaster elif node_id == max_node: config = load_ptp_config(os.path.join(PTP_CONFIG_DIR, "ptp4l_boundary.json")) write_ptp_config_to_node(hostname, "boundary", config) cmd = f"ptp4l -i eth1 -m -H -f {PTP_REMOTE_CONFIG_PATH}{BC_FILE} > {SLAVE_LOG} 2>&1 &" # Last node else: config = load_ptp_config(os.path.join(PTP_CONFIG_DIR, "ptp4l_boundary.json")) write_ptp_config_to_node(hostname, "boundary", config) cmd = f"ptp4l -i eth1 -i eth2 -m -S -f {PTP_REMOTE_CONFIG_PATH}{BC_FILE} > {BOUNDARY_LOG} 2>&1 &" # Intermediate print(f"[{hostname}] Starting ptp4l: {cmd}") try: subprocess.run(["ssh", hostname, cmd], check=True) except subprocess.CalledProcessError as e: print(f"ERROR: Failed to start ptp4l on {hostname}: {e}") def start_ptp_on_node(node_id, nodes): # New version: generate boundary clock by two ptp4l processes hostname = get_hostname(node_id) min_node = min(nodes) max_node = max(nodes) delete_logs_on_node(hostname) if node_id == min_node: # Grandmaster node config = load_ptp_config(os.path.join(PTP_CONFIG_DIR, "ptp4l_grandmaster.json")) write_ptp_config_to_node(hostname, "grandmaster", config) cmd = f"{PTP4L_INSTANCE} -i eth2 -m -H -f {PTP_REMOTE_CONFIG_PATH}{GM_FILE} > {GRANDMASTER_LOG} 2>&1 &" print(f"[{hostname}] Starting ptp4l (Grandmaster): {cmd}") subprocess.run(["ssh", hostname, cmd], check=True) elif node_id == max_node: # Last node (slave only) config = load_ptp_config(os.path.join(PTP_CONFIG_DIR, "ptp4l_bc_slave.json")) write_ptp_config_to_node(hostname, "slave", config) cmd = f"{PTP4L_INSTANCE} -i eth1 -m -H -f {PTP_REMOTE_CONFIG_PATH}{SV_FILE} > {SLAVE_LOG} 2>&1 &" print(f"[{hostname}] Starting ptp4l (Last node -> Slave only): {cmd}") subprocess.run(["ssh", hostname, cmd], check=True) else: # Intermediate nodes: ptp4l (slave) + phc2sys + ptp4l (master) config_slave = load_ptp_config(os.path.join(PTP_CONFIG_DIR, "ptp4l_bc_slave.json")) config_master = load_ptp_config(os.path.join(PTP_CONFIG_DIR, "ptp4l_bc_master.json")) # Push configs to node write_ptp_config_to_node(hostname, "slave", config_slave) write_ptp_config_to_node(hostname, "master",config_master) # Build full command chain cmd_slave = f"{PTP4L_INSTANCE} -i eth1 -m -H -f {PTP_REMOTE_CONFIG_PATH}{SV_FILE} > {SLAVE_LOG} 2>&1 &" cmd_phc2sys = f"phc2sys -s eth1 -c CLOCK_REALTIME -O 0 -m > {PHC2SYS_LOG} 2>&1 &" cmd_master = f"{PTP4L_INSTANCE} -i eth2 -m -H -f {PTP_REMOTE_CONFIG_PATH}{MA_FILE} > {MASTER_LOG} 2>&1 &" full_cmd = f"{cmd_slave} {cmd_phc2sys} {cmd_master}" print(f"[{hostname}] Starting ptp4l (Slave), phc2sys, ptp4l (Master)") subprocess.run(["ssh", hostname, full_cmd], check=True) def is_ptp_running(hostname): try: subprocess.run( ["ssh", hostname, "pgrep -x ptp4l > /dev/null"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) return True except subprocess.CalledProcessError: return False def wait_for_all_ptp_processes(nodes, retries=10, delay=2): node_hostnames = [get_hostname(n) for n in nodes] print("Checking ptp4l processes on all nodes...") for attempt in range(1, retries + 1): not_running = [] for hostname in node_hostnames: if not is_ptp_running(hostname): not_running.append(hostname) if not not_running: print("All ptp4l processes are running.") return True else: print(f"Attempt {attempt}/{retries}: Waiting for ptp4l on: {', '.join(not_running)}") time.sleep(delay) print("Error: Some ptp4l processes did not start in time.") return False def collect_logs_old(nodes, output_path): with open(output_path, "w") as outfile: for node_id in nodes: hostname = get_hostname(node_id) print(f"[{hostname}] Fetching log...") if node_id == min(nodes): # Grandmaster node log_source = {GRANDMASTER_LOG} else: # Boundary clock node log_source = {BOUNDARY_LOG} try: result = subprocess.run( ["ssh", hostname, f"cat {log_source}"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) outfile.write(f"===== BEGIN LOG FROM {hostname} =====\n") outfile.write(result.stdout) outfile.write(f"\n===== END LOG FROM {hostname} =====\n\n") except subprocess.CalledProcessError as e: print(f"ERROR: Failed to fetch log from {hostname}: {e.stderr}") outfile.write(f"===== ERROR FETCHING LOG FROM {hostname} =====\n") outfile.write(e.stderr + "\n") def collect_logs(nodes, output_path): with open(output_path, "w") as outfile: for node_id in nodes: hostname = get_hostname(node_id) print(f"[{hostname}] Fetching logs...") # Determine log sources per node role if node_id == min(nodes): # Grandmaster node log_sources = [("ptp4l", GRANDMASTER_LOG)] elif node_id == max(nodes): # Last node (slave only) log_sources = [("ptp4l_boundary", SLAVE_LOG)] else: # Intermediate node log_sources = [ ("ptp4l_slave", SLAVE_LOG), ("phc2sys", PHC2SYS_LOG), ("ptp4l_master", MASTER_LOG) ] # Fetch and write each log for log_label, log_path in log_sources: try: result = subprocess.run( ["ssh", hostname, f"sync; sleep 1; cat {log_path}"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) cleaned = result.stdout.replace('\x00', '') outfile.write(f"===== BEGIN {log_label.upper()} LOG FROM {hostname} =====\n") outfile.write(cleaned) outfile.write(f"\n===== END {log_label.upper()} LOG FROM {hostname} =====\n\n") except subprocess.CalledProcessError as e: print(f"ERROR: Failed to fetch {log_label} log from {hostname}: {e.stderr}") outfile.write(f"===== ERROR FETCHING {log_label.upper()} LOG FROM {hostname} =====\n") outfile.write(e.stderr + "\n") def stop_ptp_on_nodes(nodes): print("Stopping ptp4l on all nodes...") for node_id in nodes: hostname = get_hostname(node_id) try: subprocess.run(["ssh", hostname, "pkill -x ptp4l"], check=True) subprocess.run(["ssh", hostname, "sync"], check=True) print(f"[{hostname}] ptp4l stopped stopped and logs flushed.") except subprocess.CalledProcessError: print(f"[{hostname}] ptp4l was not running or could not be stopped.") try: subprocess.run(["ssh", hostname, "pkill -x phc2sys"], check=True) subprocess.run(["ssh", hostname, "sync"], check=True) print(f"[{hostname}] phc2sys stopped and logs flushed.") except subprocess.CalledProcessError: print(f"[{hostname}] phc2sys was not running or could not be stopped.") def main(): if len(sys.argv) < 3: usage() try: duration = int(sys.argv[1]) nodes = [int(n) for n in sys.argv[2:]] except ValueError: usage() print(f"Starting PTP experiment with nodes: {nodes}") print(f"Test duration: {duration} seconds") print(f"Killing all PTP processes on nodes...") stop_ptp_on_nodes(nodes) total_nodes = len(nodes) for index, node_id in enumerate(nodes): start_ptp_on_node(node_id, nodes) if not wait_for_all_ptp_processes(nodes): print("Aborting experiment due to startup failure.") stop_ptp_on_nodes(nodes) sys.exit(1) print(f"Running experiment for {duration} seconds...") time.sleep(duration) log_filename = generate_output_filename(nodes, duration) output_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), log_filename) print("Stopping PTP processes on all nodes...") stop_ptp_on_nodes(nodes) print(f"Collecting logs into: {output_path}") collect_logs(nodes, output_path) print("Experiment complete. Logs saved.") if __name__ == "__main__": main()