diff --git a/sonic-bmcctld/scripts/bmcctld b/sonic-bmcctld/scripts/bmcctld index e57f114e6..bc1bf725c 100644 --- a/sonic-bmcctld/scripts/bmcctld +++ b/sonic-bmcctld/scripts/bmcctld @@ -23,14 +23,12 @@ keeps STATE_DB:HOST_STATE current. Writes transitional states (POWERING_ON/OFF/CYCLING) before each call; rolls back on exception. - PolicyReader Reads admin_status, power_on_delay, and - graceful_shutdown_timeout from CHASSIS_MODULE|SWITCH-HOST in CONFIG_DB. + PolicyReader Reads bmcctld configuration from CONFIG_DB. CriticalEventChecker Checks STATE_DB for active critical system leaks or Rack Manager alerts; gates power-on decisions. - GracefulShutdownHandler Sends gNOI System.Reboot(COLD), waits for OFFLINE, - then always issues power_off() to ensure power is - removed. Falls back to immediate power_off if gNOI - fails or graceful_shutdown_timeout is 0. + GracefulShutdownHandler Sends tagged gNOI System.Reboot(HALT), evaluates + RebootStatus, then issues power_off() unless the + operation is preempted. BmcEventHandler Runs on the "bmcctld-events" thread. Subscribes to STATE_DB and enqueues ActionItems for the main thread. BmcctldDaemon Main daemon. Starts the event thread, runs the @@ -41,16 +39,22 @@ try: import json + import itertools import logging import os import queue + import re import signal import sys import threading import time + import traceback + import uuid + from dataclasses import dataclass from datetime import datetime + from enum import Enum - from sonic_py_common import daemon_base, logger + from sonic_py_common import daemon_base, device_info, logger from sonic_platform_base.chassis_base import ChassisBase if os.getenv("BMCCTLD_UNIT_TESTING") == "1": @@ -60,6 +64,10 @@ try: except ImportError as e: raise ImportError(str(e) + " - required module not found") +import grpc +from sonic_grpc.gnoi.client import GnoiClient +from sonic_grpc.gnoi import system_pb2 + CRITICAL_EVENT_LOG_FILE = "/host/bmc/event.log" @@ -127,19 +135,49 @@ class EventLogger(logger.Logger): self._file_logger.error(msg) +@dataclass(frozen=True) class ActionItem: """ Carries a power action from the BmcEventHandler thread to the BmcctldDaemon main thread action loop. - on_complete(success: bool) is called by the main thread after the + on_complete(success: bool, detail: str) is called by the main thread after the action finishes — used by Rack Manager command handlers to write the final CMD_STATUS_DONE / CMD_STATUS_FAILED back to STATE_DB. + + startup marks the automatic power-on queued at daemon start. """ - def __init__(self, action, event_desc, on_complete=None): - self.action = action - self.event_desc = event_desc - self.on_complete = on_complete + action: str + event_desc: str + priority: int + on_complete: object = None + rack_cmd_key: object = None + startup: bool = False + + +class Operation: + """Mutable state for one worker-backed power operation.""" + + def __init__(self, item, request_id, stage): + self.item = item + self.request_id = request_id + self.cancel = threading.Event() + self.cancelled_by_stop = False + self.thread = None + self.stage = stage + self.outcome = None + self.joined_callbacks = [] + self.leg_graceful = False + self.leg_reason = None + # Effective rank; a joining request of higher rank lowers it. + self.priority = item.priority + + +class PowerCallResult(Enum): + CONFIRMED = "confirmed" + NOT_CONFIRMED = "not_confirmed" + CANCELLED = "cancelled" + REFUSED_LEAK = "refused_leak" # ============================================================ @@ -184,10 +222,15 @@ FIELD_LAST_CHANGE_TIMESTAMP = "last_change_timestamp" FIELD_COMMAND = "command" FIELD_STATUS = "status" FIELD_RESULT = "result" +FIELD_REQUEST_ID = "request_id" FIELD_DEVICE_LEAK_STATUS = "device_leak_status" FIELD_SEVERITY = "severity" FIELD_LEAK = "leak" FIELD_ADMIN_STATUS = "admin_status" +FIELD_OP_RESULT = "op_result" +FIELD_OP_REASON = "op_reason" +FIELD_OP_TRIGGER = "op_trigger" +FIELD_OP_REQUEST_ID = "op_request_id" # FIELD_DEVICE_POWER_STATE: Power state values - What POWER action taken on device # Stable (final) states — written after the platform call completes @@ -211,6 +254,7 @@ SWITCH_HOST_OFFLINE = "OFFLINE" CMD_POWER_ON = "POWER_ON" CMD_POWER_OFF = "POWER_OFF" CMD_GRACEFUL_SHUT = "GRACEFUL_SHUT" +CMD_GRACEFUL_RESTART = "GRACEFUL_RESTART" CMD_POWER_CYCLE = "POWER_CYCLE" # Command execution status @@ -236,10 +280,26 @@ ALERT_SEVERITY_NORMAL = "NORMAL" # Dispatch action values (LEAK_CONTROL_POLICY and queue items) ACTION_SYSLOG_ONLY = "syslog_only" ACTION_GRACEFUL_SHUTDOWN = "graceful_shutdown" +ACTION_GRACEFUL_RESTART = "graceful_restart" ACTION_POWER_OFF = "power_off" ACTION_POWER_ON = "power_on" ACTION_POWER_CYCLE = "power_cycle" +# Worker progress markers. An *_issued stage is set immediately before the +# corresponding platform call; *_confirmed is set as soon as verification +# proves the physical postcondition. +STAGE_HANDSHAKE = "handshake" +STAGE_POWER_OFF_PENDING = "power_off_pending" +STAGE_POWER_OFF_ISSUED = "power_off_issued" +STAGE_POWER_OFF_CONFIRMED = "power_off_confirmed" +STAGE_PAUSE = "pause" +STAGE_POWER_ON_PENDING = "power_on_pending" +STAGE_POWER_ON_ISSUED = "power_on_issued" +STAGE_POWER_ON_CONFIRMED = "power_on_confirmed" +STAGE_POWER_CYCLE_PENDING = "power_cycle_pending" +STAGE_POWER_CYCLE_ISSUED = "power_cycle_issued" +STAGE_POWER_CYCLE_CONFIRMED = "power_cycle_confirmed" + # Default policy values (LEAK_CONTROL_POLICY) DEFAULT_SYSTEM_LEAK_POLICY = "enabled" DEFAULT_SYSTEM_CRITICAL_LEAK_ACTION = ACTION_POWER_OFF @@ -252,8 +312,28 @@ DEFAULT_RACK_MGR_MINOR_ALERT_ACTION = ACTION_SYSLOG_ONLY DEFAULT_POWER_ON_DELAY_SECS = 300 # liquid-cooled: allow Rack Manager time to verify liquid flow # before bmcctld auto-powers Switch-Host. Unused on air-cooled # (startup path skips _initial_power_on_sequence). -DEFAULT_SHUTDOWN_DELAY_SECS = 0 # default 0 until real GNOI graceful shutdown is supported; - # operator may override via CHASSIS_MODULE|SWITCH-HOST/graceful_shutdown_timeout +DEFAULT_SHUTDOWN_DELAY_SECS = 120 + +# Terminal operation results +OP_RESULT_SUCCESS_GRACEFUL = "SUCCESS_GRACEFUL" +OP_RESULT_SUCCESS_FORCED = "SUCCESS_FORCED" +OP_RESULT_SUCCESS = "SUCCESS" +OP_RESULT_POWER_OFF_FAILED = "POWER_OFF_FAILED" +OP_RESULT_POWER_ON_FAILED = "POWER_ON_FAILED" +OP_RESULT_OFF_LEAK_BLOCKED = "OFF_LEAK_BLOCKED" +OP_RESULT_PREEMPTED = "PREEMPTED" +OP_RESULT_ABANDONED = "ABANDONED" + +# Graceful-operation reason codes +OP_REASON_NOT_QUALIFIED = "not_qualified" +OP_REASON_TIMEOUT_ZERO = "timeout_zero" +OP_REASON_ALREADY_OFF = "already_off" +OP_REASON_PREEMPTED = "preempted" +OP_REASON_RPC_FAILURE = "rpc_failure" +OP_REASON_BACKEND_ANSWERED = "backend_answered" +OP_REASON_CHECK_FAILED = "check_failed" +OP_REASON_DEADLINE = "deadline" +OP_REASON_UNCLASSIFIED = "unclassified" # Switch-Host module name key in CHASSIS_MODULE table SWITCH_HOST_MODULE_KEY = "SWITCH-HOST" @@ -272,9 +352,21 @@ BMC_JSON_PATHS = [ "/etc/sonic/bmc.json", ] DEFAULT_SWITCH_HOST_ADDR = "169.254.100.2" +DEFAULT_SWITCH_HOST_GNMI_PORT = 8080 +GNOI_SERVER_NAME = "switch-host.bmc-link.sonic" +BMC_GNOI_TABLE = "BMC_GNOI" +DEFAULT_GNOI_CERT_PATHS = { + "ca_crt": "/etc/sonic/bmc-link/ca.crt", + "client_crt": "/etc/sonic/bmc-link/client.crt", + "client_key": "/etc/sonic/bmc-link/client.key", +} +TAG_RE = r"\[bmc-req:([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\]" SELECT_TIMEOUT_MSECS = 1000 GNOI_REQUEST_TIMEOUT_SECS = 30 +GNOI_STATUS_RPC_TIMEOUT_SECS = 10 +GNOI_STATUS_POLL_INTERVAL_SECS = 1 +RESTART_PAUSE_SECS = 10 SHUTDOWN_POLL_INTERVAL_SECS = 5 # Polling constants for verifying oper_status after power* API calls @@ -325,6 +417,132 @@ def fvs_to_dict(fvs): return dict(fvs) +class _ThreadDatabase: + """Cache each connector and table only in its owning OS thread.""" + + def __init__(self): + self._local = threading.local() + + def _connections(self): + if not hasattr(self._local, "connections"): + self._local.connections = {} + return self._local.connections + + def _tables(self): + if not hasattr(self._local, "tables"): + self._local.tables = {} + return self._local.tables + + def connection(self, db_name): + connections = self._connections() + if db_name not in connections: + connections[db_name] = daemon_base.db_connect(db_name) + return connections[db_name] + + def table(self, db_name, table_name): + key = (db_name, table_name) + tables = self._tables() + if key not in tables: + tables[key] = swsscommon.Table( + self.connection(db_name), table_name) + return tables[key] + +def action_priority(action, is_critical_trigger=False): + """Return the admission-time priority for an action.""" + if is_critical_trigger and action == ACTION_POWER_OFF: + return 0 + if is_critical_trigger and action == ACTION_GRACEFUL_SHUTDOWN: + return 1 + priorities = { + ACTION_POWER_OFF: 2, + ACTION_GRACEFUL_SHUTDOWN: 3, + ACTION_GRACEFUL_RESTART: 4, + ACTION_POWER_CYCLE: 4, + ACTION_POWER_ON: 5, + } + return priorities[action] + + +def enqueue_action(action_queue, sequence_counter, item): + """Put one ActionItem on the shared priority queue.""" + action_queue.put((item.priority, next(sequence_counter), item)) + + +def classify_report(resp, our_id): + """Classify one gNOI RebootStatus report for the current operation.""" + if re.findall(TAG_RE, resp.reason) != [our_id] or resp.active: + return "keep_waiting", None + if resp.status.message: + return "forced", OP_REASON_BACKEND_ANSWERED + if resp.method == system_pb2.HALT and \ + resp.status.status == system_pb2.RebootStatus.STATUS_SUCCESS: + return "graceful", None + return "forced", OP_REASON_CHECK_FAILED + + +class GnoiRpcError(Exception): + """Normalized gNOI transport failure exposed to operation code.""" + + +class GnoiRequester: + """One-operation mTLS wrapper around the in-process gNOI client.""" + + def __init__(self, target_addr, target_port, cert_paths, server_name): + self.target_addr = target_addr + self.target_port = target_port + self.cert_paths = cert_paths + self.server_name = server_name + self._client = None + + def open(self): + try: + with open(self.cert_paths["ca_crt"], "rb") as cert_file: + ca_cert = cert_file.read() + with open(self.cert_paths["client_key"], "rb") as key_file: + client_key = key_file.read() + with open(self.cert_paths["client_crt"], "rb") as cert_file: + client_cert = cert_file.read() + + credentials = grpc.ssl_channel_credentials( + root_certificates=ca_cert, + private_key=client_key, + certificate_chain=client_cert, + ) + client = GnoiClient( + "{}:{}".format(self.target_addr, self.target_port), + options=(("grpc.ssl_target_name_override", self.server_name),), + credentials=credentials, + ) + client.__enter__() + self._client = client + except (OSError, ValueError, grpc.RpcError) as e: + raise GnoiRpcError(str(e)) from e + + def close(self): + if self._client is None: + return + client = self._client + self._client = None + try: + client.__exit__(None, None, None) + except Exception as e: + raise GnoiRpcError(str(e)) from e + + def send_halt(self, message, timeout_secs): + request = system_pb2.RebootRequest(method=system_pb2.HALT, message=message) + try: + return self._client.system.Reboot(request, timeout=timeout_secs) + except grpc.RpcError as e: + raise GnoiRpcError(str(e)) from e + + def poll_status(self, timeout_secs): + request = system_pb2.RebootStatusRequest() + try: + return self._client.system.RebootStatus(request, timeout=timeout_secs) + except grpc.RpcError as e: + raise GnoiRpcError(str(e)) from e + + # ============================================================ # SwitchHostController # ============================================================ @@ -335,15 +553,29 @@ class SwitchHostController(EventLogger): tracks its state in the HOST_STATE STATE_DB table. """ - def __init__(self, chassis): + def __init__(self, chassis, thread_database=None): super(SwitchHostController, self).__init__(SYSLOG_IDENTIFIER, CRITICAL_EVENT_LOG_FILE) self.chassis = chassis self._switch_host_module = None - self.state_db = daemon_base.db_connect("STATE_DB") - self.config_db = daemon_base.db_connect("CONFIG_DB") - self.host_state_table = swsscommon.Table(self.state_db, HOST_STATE_TABLE) - self.chassis_module_info_table = swsscommon.Table(self.state_db, CHASSIS_MODULE_INFO_TABLE) - self.chassis_module_config_table = swsscommon.Table(self.config_db, CHASSIS_MODULE_TABLE) + self._thread_database = thread_database or _ThreadDatabase() + self.critical_event_checker = None + self._power_lock = threading.Lock() + + @property + def host_state_table(self): + return self._thread_database.table("STATE_DB", HOST_STATE_TABLE) + + @property + def chassis_module_info_table(self): + return self._thread_database.table( + "STATE_DB", CHASSIS_MODULE_INFO_TABLE) + + @property + def chassis_module_config_table(self): + return self._thread_database.table("CONFIG_DB", CHASSIS_MODULE_TABLE) + + def set_critical_event_checker(self, critical_event_checker): + self.critical_event_checker = critical_event_checker def _get_switch_host_module(self): """Return the Switch-Host Module object from the chassis module list.""" @@ -373,99 +605,170 @@ class SwitchHostController(EventLogger): # Normalise to uppercase to match HOST_STATE DB schema (ONLINE / OFFLINE) return str(raw).upper() if raw != NOT_AVAILABLE else raw - def _verify_oper_status(self, expected_status, timeout_secs, context): - """ - Poll get_oper_status() until it matches expected_status or timeout_secs elapses. - Returns True when confirmed, False on timeout. - """ + def _verify_oper_status(self, expected_status, timeout_secs, context, cancel=None): + """Poll oper status until confirmed, timed out, or cancelled.""" deadline = time.monotonic() + timeout_secs - while time.monotonic() < deadline: + while True: + if cancel is not None and cancel.is_set(): + return PowerCallResult.CANCELLED if self.get_oper_status() == expected_status: self.log_info("{}: oper_status confirmed {}".format(context, expected_status)) - return True - time.sleep(POWER_VERIFY_POLL_INTERVAL_SECS) + return PowerCallResult.CONFIRMED + remaining = deadline - time.monotonic() + if remaining <= 0: + break + interval = min(POWER_VERIFY_POLL_INTERVAL_SECS, remaining) + if cancel is not None: + if cancel.wait(interval): + return PowerCallResult.CANCELLED + else: + time.sleep(interval) self.log_error("{}: oper_status did not reach {} within {}s (current: {})".format( context, expected_status, timeout_secs, self.get_oper_status())) - return False + return PowerCallResult.NOT_CONFIRMED + + def _apply_admin_state(self, operation, cancel, admin_up, transitional_power_state, + final_power_state, expected_oper, op_name): + """Issue and verify one set_admin_state call under the power lock.""" + if cancel is not None and cancel.is_set(): + return PowerCallResult.CANCELLED - def _apply_admin_state(self, admin_up, transitional_power_state, final_power_state, expected_oper, op_name): - """ - Common implementation for power_on and power_off. - Snapshots prior STATE_DB values, writes the transitional power state, - issues set_admin_state, verifies the result, then writes the final power state. - device_status is always written from module->get_oper_status() (ONLINE|OFFLINE). - Rolls back to the prior STATE_DB snapshot on exception. - """ module = self._get_switch_host_module() if module is None: self.log_error("Cannot {} Switch-Host: module not found".format(op_name)) - return False - prior_power_state = self.get_db_power_state() - prior_status = self.get_db_device_status() + return PowerCallResult.NOT_CONFIRMED + + if admin_up: + prior_power_state = self.get_db_power_state() + prior_status = self.get_db_device_status() try: - # Write transitional power state (e.g. POWERING_ON/OFF) before platform call - self._update_host_state(transitional_power_state) - module.set_admin_state(admin_up) + try: + self._update_host_state(transitional_power_state) + except Exception as e: + if admin_up: + raise + self.log_error( + "Failed to record Switch-Host power_off transition: {}".format( + repr(e))) + refused_leak = False + with self._power_lock: + if cancel is not None and cancel.is_set(): + return PowerCallResult.CANCELLED + if admin_up and self.critical_event_checker is not None and \ + self.critical_event_checker.has_any_critical_event(): + refused_leak = True + else: + operation.stage = (STAGE_POWER_ON_ISSUED if admin_up + else STAGE_POWER_OFF_ISSUED) + module.set_admin_state(admin_up) + if refused_leak: + self.log_warning("CRITICAL_LEAK_PRESENT: refusing Switch-Host power_on") + self._update_host_state(prior_power_state, prior_status) + return PowerCallResult.REFUSED_LEAK self.log_notice("Switch-Host {} issued; verifying oper_status...".format(op_name)) - verified = self._verify_oper_status(expected_oper, POWER_VERIFY_TIMEOUT_SECS, op_name) - if not verified: + verified = self._verify_oper_status( + expected_oper, POWER_VERIFY_TIMEOUT_SECS, op_name, cancel) + if verified == PowerCallResult.CONFIRMED: + operation.stage = (STAGE_POWER_ON_CONFIRMED if admin_up + else STAGE_POWER_OFF_CONFIRMED) + elif verified == PowerCallResult.CANCELLED: + return verified + else: self.log_error("Switch-Host {}: oper_status did not confirm {}".format( op_name, expected_oper)) - # Write final stable power state (e.g. POWER_ON/OFF); device_status from live oper_status - self._update_host_state(final_power_state) + # Keep the transitional label (power is unknown); refresh the status. + self._update_host_state(transitional_power_state) + + if verified == PowerCallResult.CONFIRMED: + self._update_host_state(final_power_state) return verified except Exception as e: self.log_error("Failed to {} Switch-Host: {}".format(op_name, repr(e))) - self._update_host_state(prior_power_state, prior_status) - return False - - def power_on(self): + if cancel is not None and cancel.is_set(): + return PowerCallResult.CANCELLED + confirmed_stage = (STAGE_POWER_ON_CONFIRMED if admin_up + else STAGE_POWER_OFF_CONFIRMED) + if operation.stage == confirmed_stage: + raise + if not admin_up and operation.stage in ( + STAGE_HANDSHAKE, STAGE_POWER_OFF_PENDING): + raise + # An issued raise may still take effect: keep its transitional label. + if admin_up and operation.stage != STAGE_POWER_ON_ISSUED: + self._update_host_state(prior_power_state, prior_status) + return PowerCallResult.NOT_CONFIRMED + + def power_on(self, operation, cancel=None): """Power ON the Switch-Host via module->set_admin_state(True). device_power_state: POWERING_ON (transitional) -> POWER_ON (final) device_status: ONLINE | OFFLINE from module->get_oper_status() """ return self._apply_admin_state( - True, SWITCH_HOST_POWERING_ON, POWER_STATE_ON, SWITCH_HOST_ONLINE, "power_on") + operation, cancel, True, SWITCH_HOST_POWERING_ON, POWER_STATE_ON, + SWITCH_HOST_ONLINE, "power_on") - def power_off(self): + def power_off(self, operation, cancel=None): """Power OFF the Switch-Host via module->set_admin_state(False). device_power_state: POWERING_OFF (transitional) -> POWER_OFF (final) device_status: ONLINE | OFFLINE from module->get_oper_status() """ return self._apply_admin_state( - False, SWITCH_HOST_POWERING_OFF, POWER_STATE_OFF, SWITCH_HOST_OFFLINE, "power_off") + operation, cancel, False, SWITCH_HOST_POWERING_OFF, POWER_STATE_OFF, + SWITCH_HOST_OFFLINE, "power_off") - def power_cycle(self): + def power_cycle(self, operation, cancel=None): """Power cycle the Switch-Host via module->do_power_cycle(). device_power_state: POWER_CYCLING (transitional) -> POWER_CYCLE (final) device_status: ONLINE | OFFLINE from module->get_oper_status() """ + if cancel is not None and cancel.is_set(): + return PowerCallResult.CANCELLED module = self._get_switch_host_module() if module is None: self.log_error("Cannot power cycle Switch-Host: module not found") - return False + return PowerCallResult.NOT_CONFIRMED prior_power_state = self.get_db_power_state() prior_status = self.get_db_device_status() try: - # Write transitional power state (POWER_CYCLING) before platform call self._update_host_state(SWITCH_HOST_POWER_CYCLING) - try_get(module.do_power_cycle, default=False) + refused_leak = False + with self._power_lock: + if cancel is not None and cancel.is_set(): + return PowerCallResult.CANCELLED + if self.critical_event_checker is not None and \ + self.critical_event_checker.has_any_critical_event(): + refused_leak = True + else: + operation.stage = STAGE_POWER_CYCLE_ISSUED + try_get(module.do_power_cycle, default=False) + if refused_leak: + self.log_warning("CRITICAL_LEAK_PRESENT: refusing Switch-Host power_cycle") + self._update_host_state(prior_power_state, prior_status) + return PowerCallResult.REFUSED_LEAK self.log_notice("Switch-Host power cycle issued; verifying oper_status...") - # Use 2x timeout: host must go offline then come back online verified = self._verify_oper_status( - SWITCH_HOST_ONLINE, POWER_VERIFY_TIMEOUT_SECS * 2, "power_cycle") - if not verified: + SWITCH_HOST_ONLINE, POWER_VERIFY_TIMEOUT_SECS * 2, "power_cycle", cancel) + if verified == PowerCallResult.CONFIRMED: + operation.stage = STAGE_POWER_CYCLE_CONFIRMED + self._update_host_state(POWER_STATE_CYCLE) + elif verified == PowerCallResult.CANCELLED: + return verified + else: self.log_error("Switch-Host power cycle: oper_status did not confirm ONLINE") - # Write final stable power state; device_status from live oper_status - self._update_host_state(POWER_STATE_CYCLE) + self._update_host_state(SWITCH_HOST_POWER_CYCLING) return verified except Exception as e: self.log_error("Failed to power cycle Switch-Host: {}".format(repr(e))) - self._update_host_state(prior_power_state, prior_status) - return False + if cancel is not None and cancel.is_set(): + return PowerCallResult.CANCELLED + if operation.stage == STAGE_POWER_CYCLE_CONFIRMED: + raise + if operation.stage != STAGE_POWER_CYCLE_ISSUED: + self._update_host_state(prior_power_state, prior_status) + return PowerCallResult.NOT_CONFIRMED def _update_host_state(self, power_state, device_status=None): """Write device_power_state and device_status to HOST_STATE STATE_DB. @@ -500,21 +803,68 @@ class SwitchHostController(EventLogger): return fvs_to_dict(result[1]).get(FIELD_DEVICE_POWER_STATE, NOT_AVAILABLE) return NOT_AVAILABLE + def get_db_operation_result(self): + """Read the current operation result marker from HOST_STATE.""" + result = self.host_state_table.get(HOST_STATE_KEY) + if result and result[0]: + return fvs_to_dict(result[1]).get(FIELD_OP_RESULT, NOT_AVAILABLE) + return NOT_AVAILABLE + + def write_operation_start(self, request_id, trigger): + self.host_state_table.set(HOST_STATE_KEY, swsscommon.FieldValuePairs([ + (FIELD_OP_REQUEST_ID, request_id), + (FIELD_OP_TRIGGER, trigger), + (FIELD_OP_RESULT, "-"), + (FIELD_OP_REASON, "-"), + ])) + + def write_operation_result(self, op_result, op_reason): + self.host_state_table.set(HOST_STATE_KEY, swsscommon.FieldValuePairs([ + (FIELD_OP_RESULT, op_result), + (FIELD_OP_REASON, op_reason), + ])) + def init_host_state(self): - """Set device_power_state from live oper_status (ONLINE -> POWERED_ON, OFFLINE -> POWERED_OFF). + """Refresh device_power_state at daemon startup. - Called only at daemon startup. Any persisted transitional state from a prior - interrupted run is overwritten with the stable value derived from live oper_status. + ONLINE -> POWERED_ON. OFFLINE -> POWERED_OFF only when a power-off was + already recorded or nothing was; any other recorded state is kept, since + this daemon issued no removal. A persisted nonterminal operation is + marked abandoned first. """ + stale_state = NOT_AVAILABLE + result = self.host_state_table.get(HOST_STATE_KEY) + if result and result[0]: + entry = fvs_to_dict(result[1]) + stale_state = entry.get(FIELD_DEVICE_POWER_STATE, NOT_AVAILABLE) + request_id = entry.get(FIELD_OP_REQUEST_ID, "-") + parsed_request_id = None + try: + parsed_request_id = uuid.UUID(request_id) + except (AttributeError, ValueError): + pass + if parsed_request_id is not None and \ + str(parsed_request_id) == request_id and \ + entry.get(FIELD_OP_RESULT) in (None, "-"): + self.write_operation_result(OP_RESULT_ABANDONED, "-") + self.log_warning( + "OP_ABANDONED request_id={} stale_state={}".format( + request_id, stale_state)) + oper_status = self.get_oper_status() - power_state = POWER_STATE_ON if str(oper_status).upper() == SWITCH_HOST_ONLINE else POWER_STATE_OFF + if str(oper_status).upper() == SWITCH_HOST_ONLINE: + power_state = POWER_STATE_ON + elif stale_state not in (NOT_AVAILABLE, POWER_STATE_OFF): + power_state = stale_state + else: + power_state = POWER_STATE_OFF self._update_host_state(power_state, oper_status) def initialize_chassis_module(self, admin_status): """Populate CHASSIS_MODULE|SWITCH-HOST in both CONFIG_DB and STATE_DB at daemon startup. CONFIG_DB|CHASSIS_MODULE|SWITCH-HOST is seeded with the supplied admin_status - and default timings (power_on_delay=0, graceful_shutdown_timeout=0) only when + and default timings only when no entry exists, so any operator override persisted from a prior boot is preserved. STATE_DB|CHASSIS_MODULE_TABLE|SWITCH-HOST is always (re)populated with module identity, current oper_status, and the supplied admin_status. @@ -574,15 +924,15 @@ class SwitchHostController(EventLogger): # ============================================================ class PolicyReader(EventLogger): - """Reads bmcctld timing and leak policy configuration from CONFIG_DB.""" + """Reads bmcctld configuration from CONFIG_DB.""" - def __init__(self): + def __init__(self, thread_database=None): super(PolicyReader, self).__init__(SYSLOG_IDENTIFIER) - self.config_db = daemon_base.db_connect("CONFIG_DB") + self._thread_database = thread_database or _ThreadDatabase() def _get_chassis_module_entry(self): """Return the CHASSIS_MODULE|SWITCH-HOST entry as a dict (empty dict if missing).""" - tbl = swsscommon.Table(self.config_db, CHASSIS_MODULE_TABLE) + tbl = self._thread_database.table("CONFIG_DB", CHASSIS_MODULE_TABLE) result = tbl.get(SWITCH_HOST_MODULE_KEY) if result and result[0]: return fvs_to_dict(result[1]) @@ -598,7 +948,7 @@ class PolicyReader(EventLogger): return self._get_chassis_module_entry().get(FIELD_ADMIN_STATUS, default) def get_power_on_delay(self): - """Return power_on_delay in seconds from CHASSIS_MODULE|SWITCH-HOST (default: 0).""" + """Return the configured power_on_delay in seconds, or the default delay.""" raw = self._get_chassis_module_entry().get(FIELD_POWER_ON_DELAY) if raw is not None: try: @@ -608,7 +958,7 @@ class PolicyReader(EventLogger): return DEFAULT_POWER_ON_DELAY_SECS def get_graceful_shutdown_timeout(self): - """Return graceful_shutdown_timeout in seconds from CHASSIS_MODULE|SWITCH-HOST (default: 0).""" + """Return the configured graceful shutdown timeout, or 120 seconds.""" raw = self._get_chassis_module_entry().get(FIELD_GRACEFUL_SHUTDOWN_TIMEOUT) if raw is not None: try: @@ -617,6 +967,21 @@ class PolicyReader(EventLogger): pass return DEFAULT_SHUTDOWN_DELAY_SECS + def get_gnoi_cert_paths(self): + """Resolve the client credential paths for one operation. + + Each field comes from CONFIG_DB BMC_GNOI|certs when set, else from + its /etc/sonic/bmc-link default. Nothing else is consulted, so a BMC + without link credentials stays not_qualified instead of trying its + own gNMI server certificate; to reuse existing files, point these + fields at them. + """ + table = self._thread_database.table("CONFIG_DB", BMC_GNOI_TABLE) + result = table.get("certs") + entry = fvs_to_dict(result[1]) if result and result[0] else {} + return {field: entry.get(field, default) + for field, default in DEFAULT_GNOI_CERT_PATHS.items()} + def get_leak_control_policy(self): """Return LEAK_CONTROL_POLICY as a dict with action defaults filled in.""" policy = { @@ -627,7 +992,8 @@ class PolicyReader(EventLogger): "rack_mgr_critical_alert_action": DEFAULT_RACK_MGR_CRITICAL_ALERT_ACTION, "rack_mgr_minor_alert_action": DEFAULT_RACK_MGR_MINOR_ALERT_ACTION, } - tbl = swsscommon.Table(self.config_db, LEAK_CONTROL_POLICY_TABLE) + tbl = self._thread_database.table( + "CONFIG_DB", LEAK_CONTROL_POLICY_TABLE) # LEAK_CONTROL_POLICY is a single-row table; CLI writes key "policy", legacy variants also tried for key in ("policy", LEAK_CONTROL_POLICY_TABLE, ""): result = tbl.get(key) @@ -652,14 +1018,15 @@ class CriticalEventChecker(logger.Logger): "Rack_level_leak", ] - def __init__(self, policy_reader): + def __init__(self, policy_reader, thread_database=None): super(CriticalEventChecker, self).__init__(SYSLOG_IDENTIFIER) - self.state_db = daemon_base.db_connect("STATE_DB") + self._thread_database = thread_database or _ThreadDatabase() self.policy_reader = policy_reader def has_critical_system_leak(self): """Return True if SYSTEM_LEAK_STATUS|system shows CRITICAL_SYSTEM_LEAK.""" - tbl = swsscommon.Table(self.state_db, SYSTEM_LEAK_STATUS_TABLE) + tbl = self._thread_database.table( + "STATE_DB", SYSTEM_LEAK_STATUS_TABLE) result = tbl.get(SYSTEM_LEAK_STATUS_KEY) if result and result[0]: if fvs_to_dict(result[1]).get(FIELD_DEVICE_LEAK_STATUS) == SYSTEM_LEAK_CRITICAL: @@ -668,7 +1035,8 @@ class CriticalEventChecker(logger.Logger): def has_critical_rack_mgr_alert(self): """Return True if any RACK_MANAGER_ALERT entry has CRITICAL severity.""" - tbl = swsscommon.Table(self.state_db, RACK_MANAGER_ALERT_TABLE) + tbl = self._thread_database.table( + "STATE_DB", RACK_MANAGER_ALERT_TABLE) for alert_key in self.ALERT_KEYS: result = tbl.get(alert_key) if result and result[0]: @@ -694,7 +1062,8 @@ class CriticalEventChecker(logger.Logger): def get_system_leak_status(self): """Return device_leak_status string from SYSTEM_LEAK_STATUS table.""" - tbl = swsscommon.Table(self.state_db, SYSTEM_LEAK_STATUS_TABLE) + tbl = self._thread_database.table( + "STATE_DB", SYSTEM_LEAK_STATUS_TABLE) result = tbl.get(SYSTEM_LEAK_STATUS_KEY) if result and result[0]: return fvs_to_dict(result[1]).get(FIELD_DEVICE_LEAK_STATUS, "") @@ -707,9 +1076,8 @@ class CriticalEventChecker(logger.Logger): class GracefulShutdownHandler(EventLogger): """ - Issues a GNOI soft shutdown to Switch-Host and waits graceful_shutdown_timeout - seconds for it to go OFFLINE. Call platform power_off after the graceful_shutdown_timeout - irrespective of GNOI request succeeded or not. + Issues a tagged gNOI HALT, waits for attributed terminal evidence within + one deadline, and then removes Switch-Host power unless displaced. """ def __init__(self, controller, policy_reader): @@ -718,102 +1086,223 @@ class GracefulShutdownHandler(EventLogger): self.controller = controller self.policy_reader = policy_reader - def execute(self): - """ - Perform graceful shutdown of Switch-Host. - Sends gNOI System.Reboot(COLD) and waits for the host to go OFFLINE, - then always issues power_off() to ensure power is removed. - Returns True on success, False on failure. - """ - graceful_shutdown_timeout = self.policy_reader.get_graceful_shutdown_timeout() - self.log_notice("Initiating graceful shutdown of Switch-Host (timeout={}s)".format(graceful_shutdown_timeout)) + def _is_graceful_qualified(self, cert_paths): + """Return whether this BMC is currently qualified for the graceful leg.""" + try: + if not device_info.is_switch_bmc(): + return False + except Exception as e: + self.log_error( + "Failed to evaluate graceful shutdown qualification: {}".format(e)) + return False - prior_power_state = self.controller.get_db_power_state() - prior_status = self.controller.get_db_device_status() + try: + for field in DEFAULT_GNOI_CERT_PATHS: + path = cert_paths[field] + if not isinstance(path, str) or not os.path.isabs(path): + self.log_warning("Invalid BMC gNOI {} path: {!r}".format(field, path)) + return False + if not os.path.isfile(path) or os.path.getsize(path) == 0: + self.log_warning("Missing or empty BMC gNOI {} file: {}".format(field, path)) + return False + return True + except OSError as e: + self.log_warning("Cannot check BMC gNOI {} file {}: {}".format(field, path, e)) + return False - # Write transitional power state (GRACEFUL_SHUTTING_DOWN) before issuing shutdown + def _run_shutdown_leg(self, operation, requester_factory): + """Run one tagged graceful-shutdown handshake and retain its evidence.""" + timeout = self.policy_reader.get_graceful_shutdown_timeout() + cert_paths = self.policy_reader.get_gnoi_cert_paths() + self.log_notice( + "Initiating graceful shutdown of Switch-Host (timeout={}s)".format(timeout)) self.controller._update_host_state(SWITCH_HOST_GRACEFUL_SHUTTING_DOWN) - # graceful_shutdown_timeout == 0 means skip GNOI and force power-off immediately - if graceful_shutdown_timeout == 0: - self.log_notice("graceful_shutdown_timeout is 0: forcing immediate power OFF") + reason = None + graceful = False + requester = None + + if not self._is_graceful_qualified(cert_paths): + reason = OP_REASON_NOT_QUALIFIED + elif timeout == 0: + reason = OP_REASON_TIMEOUT_ZERO + elif self.controller.get_oper_status() == SWITCH_HOST_OFFLINE: + reason = OP_REASON_ALREADY_OFF else: + deadline = time.monotonic() + timeout + tag_message = "BMC pre-shutdown request [bmc-req:{}]".format( + operation.request_id) try: - gnoi_ok = self._issue_gnoi_shutdown() - except Exception as e: - self.log_error("Unexpected error during GNOI shutdown: {}".format(repr(e))) - self.controller._update_host_state(prior_power_state, prior_status) - return False - - if not gnoi_ok: - self.log_warning("GNOI shutdown request failed; proceeding with power OFF") - else: - # Wait up to graceful_shutdown_timeout for Switch-Host to go OFFLINE gracefully - self.log_info("Waiting up to {}s for Switch-Host to go OFFLINE...".format(graceful_shutdown_timeout)) - went_offline = self.controller._verify_oper_status( - SWITCH_HOST_OFFLINE, graceful_shutdown_timeout, "graceful_shutdown") - if went_offline: - self.log_notice("Switch-Host went OFFLINE gracefully; issuing power OFF") - else: - self.log_warning("Graceful shutdown timed out after {}s; issuing power OFF".format( - graceful_shutdown_timeout)) - - success = self.controller.power_off() - self._event_log.log_notice("GRACEFUL_SHUTDOWN done: success={} (timeout={}s)".format( - success, graceful_shutdown_timeout)) - return success - - def _issue_gnoi_shutdown(self): - # TODO: This approach of calling gnoi_client to be replaced by idea in this PR - # https://github.com/sonic-net/sonic-host-services/pull/361 - """ - Send a GNOI System.Reboot (COLD/graceful) request to the Switch-Host. - Returns True if the request was accepted, False otherwise. - Reference: https://github.com/sonic-net/SONiC/blob/master/doc/mgmt/gnmi/gnoi_system_hld.md - switch_host_addr = self._get_switch_host_addr() - self.log_info("Sending GNOI shutdown to Switch-Host at {}".format(switch_host_addr)) - try: - result = subprocess.run( - [ - "gnoi_client", - "-target", switch_host_addr, - "-rpc", "System.Reboot", - "-jsonin", '{"method": 1}', # method 1 = COLD (graceful reboot/shutdown) - ], - timeout=GNOI_REQUEST_TIMEOUT_SECS, - capture_output=True, - text=True, - ) - if result.returncode == 0: - self.log_notice("GNOI shutdown request accepted by Switch-Host") - return True - self.log_warning("GNOI shutdown failed (rc={}): {}".format(result.returncode, result.stderr)) - return False - except subprocess.TimeoutExpired: - self.log_warning("GNOI shutdown request timed out after {}s".format(GNOI_REQUEST_TIMEOUT_SECS)) - return False - except FileNotFoundError: - self.log_warning("gnoi_client not found; cannot issue graceful shutdown") - return False - except Exception as e: - self.log_warning("GNOI shutdown exception: {}".format(repr(e))) - return False - """ - return True + try: + requester = requester_factory( + self._get_switch_host_addr(), + self._get_switch_host_gnoi_port(), + cert_paths, + GNOI_SERVER_NAME, + ) + if operation.cancel.is_set(): + reason = OP_REASON_PREEMPTED + else: + requester.open() + if operation.cancel.is_set(): + reason = OP_REASON_PREEMPTED + else: + remaining = deadline - time.monotonic() + if remaining <= 0: + reason = OP_REASON_DEADLINE + else: + self.log_info( + "Sending tagged GNOI HALT request_id={}".format( + operation.request_id)) + requester.send_halt( + tag_message, + timeout_secs=min(GNOI_REQUEST_TIMEOUT_SECS, remaining), + ) + except GnoiRpcError as e: + self.log_warning("GNOI shutdown RPC failed: {}".format(str(e))) + reason = OP_REASON_RPC_FAILURE + + while reason is None and not graceful: + if operation.cancel.is_set(): + reason = OP_REASON_PREEMPTED + break + remaining = deadline - time.monotonic() + if remaining <= 0: + reason = OP_REASON_DEADLINE + break + try: + response = requester.poll_status( + timeout_secs=min(GNOI_STATUS_RPC_TIMEOUT_SECS, remaining)) + except GnoiRpcError as e: + if operation.cancel.is_set(): + reason = OP_REASON_PREEMPTED + else: + self.log_warning( + "GNOI status RPC failed: {}".format(str(e))) + reason = OP_REASON_RPC_FAILURE + break + + if operation.cancel.is_set(): + reason = OP_REASON_PREEMPTED + break + + remaining = deadline - time.monotonic() + if remaining <= 0: + reason = OP_REASON_DEADLINE + break + + classification, report_reason = classify_report( + response, operation.request_id) + if classification == "graceful": + graceful = True + operation.leg_graceful = True + self.log_notice( + "Tagged graceful completion received request_id={}".format( + operation.request_id)) + break + if classification == "forced": + reason = report_reason + self.log_notice( + "Tagged terminal failure received request_id={} reason={}".format( + operation.request_id, reason)) + break + + remaining = deadline - time.monotonic() + if remaining <= 0: + reason = OP_REASON_DEADLINE + break + operation.cancel.wait( + min(GNOI_STATUS_POLL_INTERVAL_SECS, remaining)) + finally: + if requester is not None: + try: + requester.close() + except GnoiRpcError as e: + self.log_warning("GNOI channel close failed: {}".format(str(e))) + + operation.leg_graceful = graceful + operation.leg_reason = "-" if graceful else reason + return graceful, reason + + def _power_off(self, operation, reason): + """Remove power after the handshake; return a failure outcome or None.""" + if reason == OP_REASON_PREEMPTED: + return OP_RESULT_PREEMPTED, OP_REASON_PREEMPTED, False + + power_result = self.controller.power_off(operation, operation.cancel) + if power_result == PowerCallResult.CANCELLED: + return OP_RESULT_PREEMPTED, reason, False + if power_result != PowerCallResult.CONFIRMED: + self._event_log.log_error( + "POWER_OFF_FAILED request_id={}".format(operation.request_id)) + return OP_RESULT_POWER_OFF_FAILED, reason, False + return None - def _get_switch_host_addr(self): - """Read Switch-Host address from bmc.json; fall back to hardcoded default.""" + def execute(self, operation, requester_factory): + """Run one tagged graceful-shutdown leg, then remove host power.""" + graceful, reason = self._run_shutdown_leg( + operation, requester_factory) + failure = self._power_off(operation, reason or "-") + if failure is not None: + return failure + if graceful: + return OP_RESULT_SUCCESS_GRACEFUL, "-", True + return OP_RESULT_SUCCESS_FORCED, reason or OP_REASON_UNCLASSIFIED, True + + def execute_restart(self, operation, requester_factory): + """Run the shutdown leg, pause while off, then raise host power.""" + graceful, reason = self._run_shutdown_leg( + operation, requester_factory) + + leg_reason = "-" if graceful else (reason or OP_REASON_UNCLASSIFIED) + failure = self._power_off(operation, leg_reason) + if failure is not None: + return failure + + operation.stage = STAGE_PAUSE + if operation.cancel.wait(RESTART_PAUSE_SECS): + return OP_RESULT_PREEMPTED, leg_reason, False + + power_result = self.controller.power_on(operation, operation.cancel) + if power_result == PowerCallResult.REFUSED_LEAK: + return OP_RESULT_OFF_LEAK_BLOCKED, leg_reason, False + if power_result == PowerCallResult.CANCELLED: + return OP_RESULT_PREEMPTED, leg_reason, False + if power_result != PowerCallResult.CONFIRMED: + self._event_log.log_error( + "POWER_ON_FAILED request_id={}".format(operation.request_id)) + return OP_RESULT_POWER_ON_FAILED, leg_reason, False + + if graceful: + return OP_RESULT_SUCCESS_GRACEFUL, "-", True + return OP_RESULT_SUCCESS_FORCED, leg_reason, True + + def _read_bmc_link_data(self): + """Read available bmc.json files in precedence order.""" for path in BMC_JSON_PATHS: try: with open(path) as f: data = json.load(f) - addr = data.get("bmc_if_addr") - if addr: - return addr - except (FileNotFoundError, json.JSONDecodeError, OSError): + except (json.JSONDecodeError, OSError): continue + yield data + + def _get_switch_host_addr(self): + """Read Switch-Host address from bmc.json; fall back to hardcoded default.""" + for data in self._read_bmc_link_data(): + addr = data.get("bmc_if_addr") + if addr: + return addr return DEFAULT_SWITCH_HOST_ADDR + def _get_switch_host_gnoi_port(self): + """Read the Switch-Host gNMI port from bmc.json.""" + for data in self._read_bmc_link_data(): + port = data.get("switch_host_gnmi_port") + if port is not None: + return port + return DEFAULT_SWITCH_HOST_GNMI_PORT + # ============================================================ # BmcEventHandler @@ -822,25 +1311,28 @@ class GracefulShutdownHandler(EventLogger): class BmcEventHandler(EventLogger): """ Runs in a dedicated daemon thread. Subscribes to RACK_MANAGER_COMMAND, - CHASSIS_MODULE, RACK_MANAGER_ALERT, and SYSTEM_LEAK_STATUS tables. + CHASSIS_MODULE, RACK_MANAGER_ALERT, RACK_MANAGER_DATA, and + SYSTEM_LEAK_STATUS tables. On each event it decides what action is needed and puts an ActionItem on - action_queue. All blocking power operations are executed by the main - daemon thread — this class never blocks. + action_queue. All blocking power operations run on the daemon-owned + single operation worker — this class never runs a power operation. """ - def __init__(self, action_queue, policy_reader, critical_event_checker, stop_event, - controller): + def __init__(self, action_queue, action_sequence, policy_reader, + critical_event_checker, stop_event, controller, + thread_database=None): super(BmcEventHandler, self).__init__(SYSLOG_IDENTIFIER) self._event_log = _event_logger # for critical leak / alert events only self.action_queue = action_queue + self.action_sequence = action_sequence self.policy_reader = policy_reader self.critical_event_checker = critical_event_checker self.stop_event = stop_event self.controller = controller - - self.state_db = daemon_base.db_connect("STATE_DB") - self.config_db = daemon_base.db_connect("CONFIG_DB") + self._thread_database = thread_database or _ThreadDatabase() + self._subscription_ready = threading.Event() + self._subscription_error = None # Act on RACK_MANAGER_ALERT severity changes only. # Rack Manager (redfish docker) may republish identical rows each cycle. @@ -849,38 +1341,47 @@ class BmcEventHandler(EventLogger): # Act on CHASSIS_MODULE admin_status transitions only. self._last_chassis_module_admin_status = {} - # One SubscriberStateTable per watched table - self.rack_mgr_cmd_sub = swsscommon.SubscriberStateTable( - self.state_db, RACK_MANAGER_COMMAND_TABLE) - self.chassis_module_sub = swsscommon.SubscriberStateTable( - self.config_db, CHASSIS_MODULE_TABLE) - self.rack_mgr_alert_sub = swsscommon.SubscriberStateTable( - self.state_db, RACK_MANAGER_ALERT_TABLE) - self.rack_mgr_data_sub = swsscommon.SubscriberStateTable( - self.state_db, RACK_MANAGER_DATA_TABLE) - self.system_leak_sub = swsscommon.SubscriberStateTable( - self.state_db, SYSTEM_LEAK_STATUS_TABLE) - - # Map fd -> (table, handler). - # getFd() returns a stable integer regardless of how swsscommon re-wraps - # the C++ Selectable* returned by sel.select(). The table reference is - # kept here because sel.select() returns a raw Selectable that has - # getFd() but no pop(); pop() must be called on the original - # SubscriberStateTable object. - self._subscribers = { - self.rack_mgr_cmd_sub.getFd(): (self.rack_mgr_cmd_sub, self._handle_rack_mgr_command), - self.chassis_module_sub.getFd(): (self.chassis_module_sub, self._handle_chassis_module), - self.rack_mgr_alert_sub.getFd(): (self.rack_mgr_alert_sub, self._handle_rack_mgr_alert), - self.rack_mgr_data_sub.getFd(): (self.rack_mgr_data_sub, self._handle_rack_mgr_data), - self.system_leak_sub.getFd(): (self.system_leak_sub, self._handle_system_leak), + def _create_event_subscriptions(self): + """Create all subscriber objects on the event thread.""" + state_db = daemon_base.db_connect("STATE_DB") + config_db = daemon_base.db_connect("CONFIG_DB") + rack_mgr_cmd_sub = swsscommon.SubscriberStateTable( + state_db, RACK_MANAGER_COMMAND_TABLE) + chassis_module_sub = swsscommon.SubscriberStateTable( + config_db, CHASSIS_MODULE_TABLE) + rack_mgr_alert_sub = swsscommon.SubscriberStateTable( + state_db, RACK_MANAGER_ALERT_TABLE) + rack_mgr_data_sub = swsscommon.SubscriberStateTable( + state_db, RACK_MANAGER_DATA_TABLE) + system_leak_sub = swsscommon.SubscriberStateTable( + state_db, SYSTEM_LEAK_STATUS_TABLE) + # Select can return a wrapper without pop(); recover the original + # subscriber by stable fd and retain its source connector with it. + subscribers = { + rack_mgr_cmd_sub.getFd(): ( + rack_mgr_cmd_sub, self._handle_rack_mgr_command, state_db), + chassis_module_sub.getFd(): ( + chassis_module_sub, self._handle_chassis_module, config_db), + rack_mgr_alert_sub.getFd(): ( + rack_mgr_alert_sub, self._handle_rack_mgr_alert, state_db), + rack_mgr_data_sub.getFd(): ( + rack_mgr_data_sub, self._handle_rack_mgr_data, state_db), + system_leak_sub.getFd(): ( + system_leak_sub, self._handle_system_leak, state_db), } - - self.sel = swsscommon.Select() - self.sel.addSelectable(self.rack_mgr_cmd_sub) - self.sel.addSelectable(self.chassis_module_sub) - self.sel.addSelectable(self.rack_mgr_alert_sub) - self.sel.addSelectable(self.rack_mgr_data_sub) - self.sel.addSelectable(self.system_leak_sub) + selector = swsscommon.Select() + for subscriber, _, _ in subscribers.values(): + selector.addSelectable(subscriber) + return selector, subscribers + + def wait_for_event_subscriptions(self): + """Wait for event subscriptions to initialize or raise their error.""" + while not self._subscription_ready.wait(1.0): + if self.stop_event.is_set(): + return False + if self._subscription_error is not None: + raise self._subscription_error + return not self.stop_event.is_set() def seed_chassis_module_admin_status(self): """ @@ -902,20 +1403,27 @@ class BmcEventHandler(EventLogger): def run_event_loop(self): """Block in select loop until stop_event is set.""" + try: + selector, subscribers = self._create_event_subscriptions() + except Exception as error: + self._subscription_error = error + self._subscription_ready.set() + return + self._subscription_ready.set() self.log_info("Entering event subscription loop") while not self.stop_event.is_set(): - state, selectable = self.sel.select(SELECT_TIMEOUT_MSECS) + state, selectable = selector.select(SELECT_TIMEOUT_MSECS) if state == swsscommon.Select.TIMEOUT: continue if state != swsscommon.Select.OBJECT: self.log_warning("sel.select() returned unexpected state: {}".format(state)) continue fd = selectable.getFd() - entry = self._subscribers.get(fd) + entry = subscribers.get(fd) if entry is None: self.log_warning("sel.select() returned unknown fd={}".format(fd)) continue - table, handler = entry + table, handler, _ = entry key, op, fvs = table.pop() if op != "SET": continue @@ -924,7 +1432,7 @@ class BmcEventHandler(EventLogger): # ---- internal event handlers ---- def _handle_rack_mgr_command(self, key, fvs): - """Process a RACK_MANAGER_COMMAND entry (POWER_ON / POWER_OFF / POWER_CYCLE).""" + """Process one supported RACK_MANAGER_COMMAND entry.""" command = fvs.get(FIELD_COMMAND, "") cmd_status = fvs.get(FIELD_STATUS, "") @@ -935,35 +1443,40 @@ class BmcEventHandler(EventLogger): self.log_notice("Rack Manager command: key={} command={}".format(key, command)) self._set_cmd_status(key, CMD_STATUS_IN_PROGRESS) - if command == CMD_POWER_ON: - if self.critical_event_checker.has_any_critical_event(): - self._event_log.log_warning( - "RACK_MGR_CMD FAILED: key={} command={} reason=CRITICAL_LEAK_PRESENT " - "(active critical leak or alert)".format(key, command)) - self._set_cmd_status(key, CMD_STATUS_FAILED, "CRITICAL_LEAK_PRESENT") - return - - elif command not in (CMD_POWER_OFF, CMD_GRACEFUL_SHUT, CMD_POWER_CYCLE): - self._event_log.log_warning( - "RACK_MGR_CMD FAILED: key={} command={} reason=UNKNOWN_COMMAND".format(key, command)) - self._set_cmd_status(key, CMD_STATUS_FAILED, "UNKNOWN_COMMAND") - return - action_map = { CMD_POWER_OFF: ACTION_POWER_OFF, CMD_GRACEFUL_SHUT: ACTION_GRACEFUL_SHUTDOWN, + CMD_GRACEFUL_RESTART: ACTION_GRACEFUL_RESTART, CMD_POWER_ON: ACTION_POWER_ON, CMD_POWER_CYCLE: ACTION_POWER_CYCLE, } + if command not in action_map: + self._event_log.log_warning( + "RACK_MGR_CMD FAILED: key={} command={} reason=UNKNOWN_COMMAND".format(key, command)) + self._set_cmd_status(key, CMD_STATUS_FAILED, "UNKNOWN_COMMAND") + return - def on_complete(success, cmd_key=key, cmd=command): + if command in (CMD_POWER_ON, CMD_POWER_CYCLE): + if self.critical_event_checker.has_any_critical_event(): + self._event_log.log_warning( + "RACK_MGR_CMD FAILED: key={} command={} reason=CRITICAL_LEAK_PRESENT " + "(active critical leak or alert)".format(key, command)) + self._set_cmd_status(key, CMD_STATUS_FAILED, "CRITICAL_LEAK_PRESENT") + return + + def on_complete(success, detail="", cmd_key=key): self._set_cmd_status(cmd_key, CMD_STATUS_DONE if success else CMD_STATUS_FAILED, - "SUCCESS" if success else "ERROR") + "SUCCESS" if success else (detail or "ERROR")) - self.action_queue.put(ActionItem(action_map[command], - "RACK_MGR_CMD:{}".format(command), - on_complete)) + action = action_map[command] + self._enqueue(ActionItem( + action, + "RACK_MGR_CMD:{}".format(command), + action_priority(action), + on_complete=on_complete, + rack_cmd_key=key, + )) def _handle_chassis_module(self, key, fvs): """Process CHASSIS_MODULE admin_status change (admin up/down from CLI).""" @@ -980,12 +1493,14 @@ class BmcEventHandler(EventLogger): self.controller.update_chassis_module_admin_status(admin_status) if admin_status == ADMIN_DOWN: - if self.controller.get_db_device_status() != SWITCH_HOST_ONLINE: - self._event_log.log_notice( - "CHASSIS_MODULE admin_down: key={} NOOP (Switch-Host already OFFLINE)".format(key)) - return - self.log_notice("CHASSIS_MODULE admin_down: initiating graceful shutdown of Switch-Host") - self.action_queue.put(ActionItem(ACTION_GRACEFUL_SHUTDOWN, "CHASSIS_MODULE:ADMIN_DOWN")) + # Common admission handles already-offline and in-flight power changes. + self.log_notice( + "CHASSIS_MODULE admin_down: enqueueing graceful shutdown request") + self._enqueue(ActionItem( + ACTION_GRACEFUL_SHUTDOWN, + "CHASSIS_MODULE:ADMIN_DOWN", + action_priority(ACTION_GRACEFUL_SHUTDOWN), + )) elif admin_status == ADMIN_UP: if self.controller.get_db_device_status() == SWITCH_HOST_ONLINE: @@ -998,7 +1513,11 @@ class BmcEventHandler(EventLogger): "(CRITICAL_LEAK_PRESENT: active critical leak or alert)".format(key)) return self.log_notice("CHASSIS_MODULE admin_up: powering ON Switch-Host") - self.action_queue.put(ActionItem(ACTION_POWER_ON, "CHASSIS_MODULE:ADMIN_UP")) + self._enqueue(ActionItem( + ACTION_POWER_ON, + "CHASSIS_MODULE:ADMIN_UP", + action_priority(ACTION_POWER_ON), + )) def _handle_system_leak(self, key, fvs): """Process SYSTEM_LEAK_STATUS change from thermalctld.""" @@ -1018,7 +1537,8 @@ class BmcEventHandler(EventLogger): action = policy.get("system_critical_leak_action", DEFAULT_SYSTEM_CRITICAL_LEAK_ACTION) self._event_log.log_notice("CRITICAL SYSTEM LEAK detected (status={}); " "action={}".format(leak_status, action)) - self._dispatch_action(action, "CRITICAL_SYSTEM_LEAK") + self._dispatch_action( + action, "CRITICAL_SYSTEM_LEAK", is_critical_trigger=True) elif leak_status == SYSTEM_LEAK_MINOR: action = policy.get("system_minor_leak_action", DEFAULT_SYSTEM_MINOR_LEAK_ACTION) @@ -1051,7 +1571,11 @@ class BmcEventHandler(EventLogger): action = policy.get("rack_mgr_critical_alert_action", DEFAULT_RACK_MGR_CRITICAL_ALERT_ACTION) self._event_log.log_notice("CRITICAL RACK MGR ALERT (severity={} key={}); " "action={}".format(severity, key, action)) - self._dispatch_action(action, "RACK_MGR_CRITICAL_EVENT key={}".format(key)) + self._dispatch_action( + action, + "RACK_MGR_CRITICAL_EVENT key={}".format(key), + is_critical_trigger=True, + ) elif severity in (ALERT_SEVERITY_MINOR, ALERT_SEVERITY_MAJOR): action = policy.get("rack_mgr_minor_alert_action", DEFAULT_RACK_MGR_MINOR_ALERT_ACTION) @@ -1089,7 +1613,7 @@ class BmcEventHandler(EventLogger): return # NORMAL telemetry: clear the matching alert if one is currently active. - tbl = swsscommon.Table(self.state_db, RACK_MANAGER_ALERT_TABLE) + tbl = self._thread_database.table("STATE_DB", RACK_MANAGER_ALERT_TABLE) result = tbl.get(key) if not (result and result[0]): # No active alert for this key; nothing to clear (avoids per-cycle noise). @@ -1102,7 +1626,10 @@ class BmcEventHandler(EventLogger): # ---- dispatch helpers ---- - def _dispatch_action(self, action, event_desc): + def _enqueue(self, item): + enqueue_action(self.action_queue, self.action_sequence, item) + + def _dispatch_action(self, action, event_desc, is_critical_trigger=False): """ Enqueue a power action for the main thread. syslog_only is handled inline; all other actions go on action_queue. @@ -1111,14 +1638,19 @@ class BmcEventHandler(EventLogger): if action == ACTION_SYSLOG_ONLY: return if action in (ACTION_GRACEFUL_SHUTDOWN, ACTION_POWER_OFF, ACTION_POWER_ON): - self.action_queue.put(ActionItem(action, event_desc)) + self._enqueue(ActionItem( + action, + event_desc, + action_priority(action, is_critical_trigger), + )) else: self.log_warning("Unknown action '{}' for event '{}'; defaulting to syslog_only".format( action, event_desc)) def _set_cmd_status(self, cmd_id, status, result="SUCCESS"): """Update RACK_MANAGER_COMMAND entry with execution status.""" - tbl = swsscommon.Table(self.state_db, RACK_MANAGER_COMMAND_TABLE) + tbl = self._thread_database.table( + "STATE_DB", RACK_MANAGER_COMMAND_TABLE) fvs = swsscommon.FieldValuePairs([ (FIELD_STATUS, status), (FIELD_RESULT, result), @@ -1126,6 +1658,394 @@ class BmcEventHandler(EventLogger): ]) tbl.set(cmd_id, fvs) + def _set_cmd_request_id(self, cmd_id, request_id): + tbl = self._thread_database.table( + "STATE_DB", RACK_MANAGER_COMMAND_TABLE) + tbl.set(cmd_id, swsscommon.FieldValuePairs([ + (FIELD_REQUEST_ID, request_id), + (FIELD_LAST_CHANGE_TIMESTAMP, get_current_timestamp()), + ])) + + +# ============================================================ +# OperationRunner +# ============================================================ + +class OperationRunner(EventLogger): + """Own the single in-flight operation and all admission decisions.""" + + INITIAL_STAGES = { + ACTION_GRACEFUL_SHUTDOWN: STAGE_HANDSHAKE, + ACTION_GRACEFUL_RESTART: STAGE_HANDSHAKE, + ACTION_POWER_OFF: STAGE_POWER_OFF_PENDING, + ACTION_POWER_ON: STAGE_POWER_ON_PENDING, + ACTION_POWER_CYCLE: STAGE_POWER_CYCLE_PENDING, + } + + def __init__(self, daemon, action_queue, action_sequence): + super(OperationRunner, self).__init__(SYSLOG_IDENTIFIER) + self._event_log = _event_logger + self.daemon = daemon + self.action_queue = action_queue + self.action_sequence = action_sequence + self.current = None + self.power_removal_admitted = False + + def enqueue(self, item): + enqueue_action(self.action_queue, self.action_sequence, item) + + def _new_request_id(self): + return str(uuid.uuid4()) + + def _write_rack_request_id(self, cmd_key, request_id): + if cmd_key: + self.daemon.event_handler._set_cmd_request_id(cmd_key, request_id) + + def _record_start(self, request_id, item): + self.daemon.controller.write_operation_start(request_id, item.event_desc) + self._write_rack_request_id(item.rack_cmd_key, request_id) + + def _record_terminal(self, item, request_id, op_result, op_reason, detail): + self.daemon.controller.write_operation_result(op_result, op_reason) + self._event_log.log_notice( + "OP_DONE action={} result={} reason={} request_id={} trigger={} detail={}".format( + item.action, op_result, op_reason, request_id, + item.event_desc, detail)) + + def _fire(self, callback, cmd_key, request_id, success, detail=""): + if callback is None: + return + try: + callback(success, detail) + except Exception: + self.log_error( + "CALLBACK_FAILED request_id={} cmd_key={}".format( + request_id or "-", cmd_key or "-")) + + def _note_admission(self, item): + if item.action in (ACTION_GRACEFUL_SHUTDOWN, ACTION_POWER_OFF): + self.power_removal_admitted = True + + def _guard_should_skip(self, item): + current_status = self.daemon.controller.get_oper_status() + db_power_state = self.daemon.controller.get_db_power_state() + if item.action in (ACTION_GRACEFUL_SHUTDOWN, ACTION_POWER_OFF): + op_result = self.daemon.controller.get_db_operation_result() + # Live OFFLINE proves a redundant repeat only with a recorded + # power-off; any other label may still be powered. + live_offline = (current_status == SWITCH_HOST_OFFLINE and + db_power_state == POWER_STATE_OFF) + recorded = db_power_state in ( + SWITCH_HOST_POWERING_OFF, + SWITCH_HOST_GRACEFUL_SHUTTING_DOWN, + ) and op_result == "-" + return live_offline or (recorded and item.priority >= 2) + if item.action == ACTION_POWER_ON: + return (current_status == SWITCH_HOST_ONLINE or + (db_power_state == SWITCH_HOST_POWERING_ON and + self.daemon.controller.get_db_operation_result() == "-")) + return False + + def _complete_guarded(self, item): + request_id = self._new_request_id() + self._record_start(request_id, item) + if item.action == ACTION_GRACEFUL_SHUTDOWN: + op_result = OP_RESULT_SUCCESS_FORCED + op_reason = OP_REASON_ALREADY_OFF + else: + op_result = OP_RESULT_SUCCESS + op_reason = "-" + if item.action == ACTION_POWER_ON: + stable_state = POWER_STATE_ON + expected_status = SWITCH_HOST_ONLINE + else: + stable_state = POWER_STATE_OFF + expected_status = SWITCH_HOST_OFFLINE + if self.daemon.controller.get_oper_status() == expected_status: + self.daemon.controller._update_host_state( + stable_state, expected_status) + self._record_terminal( + item, request_id, op_result, op_reason, "guard_skip") + self._fire( + item.on_complete, item.rack_cmd_key, request_id, True, "guard_skip") + + def _spawn(self, item): + request_id = self._new_request_id() + operation = Operation( + item, request_id, self.INITIAL_STAGES[item.action]) + self._record_start(request_id, item) + self.current = operation + operation.thread = threading.Thread( + target=self._run_worker, + args=(operation,), + daemon=True, + name="bmcctld-op", + ) + self._event_log.log_notice( + "OP_START action={} request_id={} trigger={}".format( + item.action, request_id, item.event_desc)) + operation.thread.start() + + def _plain_outcome(self, action, result): + if result == PowerCallResult.CONFIRMED: + return OP_RESULT_SUCCESS, "-", True + if result == PowerCallResult.CANCELLED: + return OP_RESULT_PREEMPTED, "-", False + if result == PowerCallResult.REFUSED_LEAK: + return OP_RESULT_OFF_LEAK_BLOCKED, "-", False + if action == ACTION_POWER_OFF: + return OP_RESULT_POWER_OFF_FAILED, "-", False + return OP_RESULT_POWER_ON_FAILED, "-", False + + def _run_worker(self, operation): + try: + action = operation.item.action + if action == ACTION_GRACEFUL_SHUTDOWN: + operation.outcome = self.daemon.graceful_shutdown.execute( + operation, self.daemon.gnoi_requester_factory) + elif action == ACTION_GRACEFUL_RESTART: + operation.outcome = self.daemon.graceful_shutdown.execute_restart( + operation, self.daemon.gnoi_requester_factory) + elif action == ACTION_POWER_OFF: + operation.outcome = self._plain_outcome( + action, + self.daemon.controller.power_off(operation, operation.cancel), + ) + elif action == ACTION_POWER_ON: + operation.outcome = self._plain_outcome( + action, + self.daemon.controller.power_on(operation, operation.cancel), + ) + elif action == ACTION_POWER_CYCLE: + operation.outcome = self._plain_outcome( + action, + self.daemon.controller.power_cycle(operation, operation.cancel), + ) + else: + raise ValueError("Unsupported operation action: {}".format(action)) + except Exception as e: + self.log_error( + "Operation worker failed action={} request_id={} stage={}: {}\n{}".format( + operation.item.action, operation.request_id, + operation.stage, repr(e), traceback.format_exc())) + operation.outcome = self._recover_worker_exception(operation) + + def _preempt_reason(self, operation): + if operation.leg_graceful: + return "-" + return operation.leg_reason or OP_REASON_PREEMPTED + + def _truthful_confirmed_outcome(self, operation): + action = operation.item.action + if action in (ACTION_GRACEFUL_SHUTDOWN, ACTION_GRACEFUL_RESTART): + if operation.leg_graceful: + return OP_RESULT_SUCCESS_GRACEFUL, "-", True + return (OP_RESULT_SUCCESS_FORCED, + operation.leg_reason or OP_REASON_UNCLASSIFIED, True) + return OP_RESULT_SUCCESS, "-", True + + def _recovery_power_off(self, operation): + """Attempt the one recovery-owned power-off without ever retrying it.""" + try: + return self.daemon.controller.power_off( + operation, operation.cancel) + except Exception as e: + self.log_error( + "Recovery power_off failed request_id={} stage={}: {}\n{}".format( + operation.request_id, operation.stage, + repr(e), traceback.format_exc())) + if operation.cancel.is_set(): + return PowerCallResult.CANCELLED + if operation.stage == STAGE_POWER_OFF_CONFIRMED: + return PowerCallResult.CONFIRMED + return PowerCallResult.NOT_CONFIRMED + + def _recover_worker_exception(self, operation): + action = operation.item.action + stage = operation.stage + + if operation.cancel.is_set(): + return OP_RESULT_PREEMPTED, self._preempt_reason(operation), False + + terminal_confirmed = ( + (action in (ACTION_GRACEFUL_SHUTDOWN, ACTION_POWER_OFF) and + stage == STAGE_POWER_OFF_CONFIRMED) or + (action in (ACTION_GRACEFUL_RESTART, ACTION_POWER_ON) and + stage == STAGE_POWER_ON_CONFIRMED) or + (action == ACTION_POWER_CYCLE and + stage == STAGE_POWER_CYCLE_CONFIRMED) + ) + if terminal_confirmed: + return self._truthful_confirmed_outcome(operation) + + if action in (ACTION_GRACEFUL_SHUTDOWN, ACTION_GRACEFUL_RESTART): + if stage == STAGE_HANDSHAKE: + result = self._recovery_power_off(operation) + if operation.cancel.is_set() or result == PowerCallResult.CANCELLED: + return (OP_RESULT_PREEMPTED, + self._preempt_reason(operation), False) + if result == PowerCallResult.CONFIRMED: + if action == ACTION_GRACEFUL_SHUTDOWN: + return (OP_RESULT_SUCCESS_FORCED, + OP_REASON_UNCLASSIFIED, True) + return (OP_RESULT_POWER_ON_FAILED, + OP_REASON_UNCLASSIFIED, False) + return (OP_RESULT_POWER_OFF_FAILED, + OP_REASON_UNCLASSIFIED, False) + if stage == STAGE_POWER_OFF_ISSUED: + return (OP_RESULT_POWER_OFF_FAILED, + OP_REASON_UNCLASSIFIED, False) + if action == ACTION_GRACEFUL_RESTART and stage in ( + STAGE_POWER_OFF_CONFIRMED, STAGE_PAUSE, + STAGE_POWER_ON_ISSUED): + return (OP_RESULT_POWER_ON_FAILED, + operation.leg_reason or OP_REASON_UNCLASSIFIED, False) + + if action == ACTION_POWER_OFF: + if stage == STAGE_POWER_OFF_PENDING: + result = self._recovery_power_off(operation) + if operation.cancel.is_set() or result == PowerCallResult.CANCELLED: + return OP_RESULT_PREEMPTED, OP_REASON_PREEMPTED, False + if result == PowerCallResult.CONFIRMED: + return OP_RESULT_SUCCESS, "-", True + return (OP_RESULT_POWER_OFF_FAILED, + OP_REASON_UNCLASSIFIED, False) + if stage == STAGE_POWER_OFF_ISSUED: + return (OP_RESULT_POWER_OFF_FAILED, + OP_REASON_UNCLASSIFIED, False) + + if action == ACTION_POWER_CYCLE and stage in ( + STAGE_POWER_CYCLE_PENDING, STAGE_POWER_CYCLE_ISSUED): + return OP_RESULT_POWER_ON_FAILED, OP_REASON_UNCLASSIFIED, False + + if action == ACTION_POWER_ON and stage in ( + STAGE_POWER_ON_PENDING, STAGE_POWER_ON_ISSUED): + return OP_RESULT_POWER_ON_FAILED, "-", False + + return OP_RESULT_ABANDONED, OP_REASON_UNCLASSIFIED, False + + def _reap_if_done(self): + operation = self.current + if operation is None or operation.thread.is_alive(): + return False + + if operation.outcome is None: + self.log_error( + "Operation worker exited without outcome request_id={}".format( + operation.request_id)) + operation.outcome = ( + OP_RESULT_ABANDONED, OP_REASON_UNCLASSIFIED, False) + if operation.cancelled_by_stop and \ + operation.outcome[0] == OP_RESULT_PREEMPTED: + # Cancelled by the daemon stop, not displaced by a request. + operation.outcome = (OP_RESULT_ABANDONED, "-", False) + + op_result, op_reason, success = operation.outcome + callbacks = [ + (operation.item.on_complete, operation.item.rack_cmd_key) + ] + operation.joined_callbacks + detail = ("CRITICAL_LEAK_PRESENT" + if op_result == OP_RESULT_OFF_LEAK_BLOCKED else op_result) + self._record_terminal( + operation.item, operation.request_id, + op_result, op_reason, op_result) + request_id = operation.request_id + self.current = None + for callback, cmd_key in callbacks: + self._fire(callback, cmd_key, request_id, success, detail) + return True + + def _displace_and_requeue(self, new_entry): + operation = self.current + new_item = new_entry[2] + self._event_log.log_notice( + "PREEMPTING request_id={} priority={} with action={} priority={}".format( + operation.request_id, operation.priority, + new_item.action, new_item.priority)) + operation.cancel.set() + next_warning = time.monotonic() + 30 + while operation.thread.is_alive(): + if self.daemon.stop_event.is_set(): + self.action_queue.put(new_entry) + return + operation.thread.join(timeout=1) + if operation.thread.is_alive() and time.monotonic() >= next_warning: + self.log_warning( + "Still joining preempted operation request_id={}".format( + operation.request_id)) + next_warning = time.monotonic() + 30 + + if operation.outcome is None or \ + operation.outcome[0] == OP_RESULT_PREEMPTED: + reason = (operation.outcome[1] + if operation.outcome and operation.outcome[1] not in (None, "-") + else self._preempt_reason(operation)) + operation.outcome = OP_RESULT_PREEMPTED, reason, False + + self._reap_if_done() + self.action_queue.put(new_entry) + + def process_next(self, timeout=1.0): + self._reap_if_done() + try: + entry = self.action_queue.get(timeout=timeout) + except queue.Empty: + return False + + self._reap_if_done() + item = entry[2] + if item.startup and self.power_removal_admitted: + # Running it now would undo the removal admitted ahead of it. + self._event_log.log_notice( + "STARTUP power_on dropped: a power-removing request was " + "admitted first") + return True + if self.current is None: + self._note_admission(item) + if self._guard_should_skip(item): + self._complete_guarded(item) + else: + self._spawn(item) + return True + + # Identical actions join before priority is compared; the joiner + # lends its rank. + if item.action == self.current.item.action: + self._note_admission(item) + if item.priority < self.current.priority: + self.current.priority = item.priority + self._event_log.log_notice( + "JOIN raised priority request_id={} priority={}".format( + self.current.request_id, item.priority)) + if item.on_complete is not None: + self.current.joined_callbacks.append( + (item.on_complete, item.rack_cmd_key)) + self._write_rack_request_id( + item.rack_cmd_key, self.current.request_id) + self._event_log.log_notice( + "JOIN action={} request_id={}".format( + item.action, self.current.request_id)) + return True + + if item.priority < self.current.priority: + self._displace_and_requeue(entry) + return True + + self._event_log.log_notice( + "REFUSED_BUSY action={} running_request_id={}".format( + item.action, self.current.request_id)) + self._fire(item.on_complete, item.rack_cmd_key, "-", False, "BUSY") + return True + + def stop(self): + operation = self.current + if operation is not None and operation.thread.is_alive(): + if not operation.cancel.is_set(): + operation.cancelled_by_stop = True + operation.cancel.set() + operation.thread.join(timeout=5) + self._reap_if_done() + # ============================================================ # BmcctldDaemon @@ -1157,17 +2077,27 @@ class BmcctldDaemon(daemon_base.DaemonBase): self.log_error("Failed to load platform chassis: {}".format(repr(e))) sys.exit(CHASSIS_LOAD_ERROR) - self.controller = SwitchHostController(self.chassis) - self.policy_reader = PolicyReader() - self.critical_event_checker = CriticalEventChecker(self.policy_reader) + self._thread_database = _ThreadDatabase() + self.controller = SwitchHostController( + self.chassis, self._thread_database) + self.policy_reader = PolicyReader(self._thread_database) + self.critical_event_checker = CriticalEventChecker( + self.policy_reader, self._thread_database) + self.controller.set_critical_event_checker(self.critical_event_checker) self.graceful_shutdown = GracefulShutdownHandler(self.controller, self.policy_reader) - self.action_queue = queue.Queue() + self.gnoi_requester_factory = GnoiRequester + self.action_queue = queue.PriorityQueue() + self.action_sequence = itertools.count() + self.operation_runner = OperationRunner( + self, self.action_queue, self.action_sequence) self.event_handler = BmcEventHandler( self.action_queue, + self.action_sequence, self.policy_reader, self.critical_event_checker, self.stop_event, self.controller, + self._thread_database, ) # Populate CHASSIS_MODULE_INFO_TABLE at startup with module identity and status. @@ -1200,11 +2130,12 @@ class BmcctldDaemon(daemon_base.DaemonBase): def _rack_mgr_power_cmd_executed(self): """ - Return True if any RACK_MANAGER_COMMAND entry for POWER_ON or POWER_OFF - has status IN_PROGRESS or DONE — indicating the Rack Manager issued an - explicit power directive during the boot delay. + Return True if a POWER_ON, POWER_OFF, GRACEFUL_SHUT, or + GRACEFUL_RESTART command has status IN_PROGRESS or DONE during the + boot delay. """ - tbl = swsscommon.Table(self.event_handler.state_db, RACK_MANAGER_COMMAND_TABLE) + tbl = self._thread_database.table( + "STATE_DB", RACK_MANAGER_COMMAND_TABLE) for key in tbl.getKeys(): result = tbl.get(key) if not (result and result[0]): @@ -1212,7 +2143,8 @@ class BmcctldDaemon(daemon_base.DaemonBase): entry = fvs_to_dict(result[1]) command = entry.get(FIELD_COMMAND, "") status = entry.get(FIELD_STATUS, "") - if command in (CMD_POWER_ON, CMD_POWER_OFF, CMD_GRACEFUL_SHUT) and \ + if command in (CMD_POWER_ON, CMD_POWER_OFF, CMD_GRACEFUL_SHUT, + CMD_GRACEFUL_RESTART) and \ status in (CMD_STATUS_IN_PROGRESS, CMD_STATUS_DONE): return True return False @@ -1223,13 +2155,15 @@ class BmcctldDaemon(daemon_base.DaemonBase): 1. Check CHASSIS_MODULE admin_status for SWITCH-HOST. If 'down' (the default), Switch-Host stays powered off — exit immediately. The operator must run 'config chassis modules startup SWITCH-HOST' to bring it up. - 2. Sleep SWITCH_HOST_POWER_ON_DELAY seconds to allow Rack Manager to come - up and verify liquid flow (liquid-cooled) or simply stabilise (air-cooled). - While sleeping, keep draining the action_queue so Rack Manager commands + 2. On a cold boot, wait for the configured power-on delay minus system + uptime, allowing Rack Manager to start and verify liquid flow. + While waiting, keep draining the action_queue so Rack Manager commands (e.g. POWER_OFF) arriving during the delay are acted on immediately. - 3. After the delay, check if Rack Manager issued an explicit POWER_ON, - POWER_OFF, or GRACEFUL_SHUT command (CMD_STATUS_IN_PROGRESS or DONE in - RACK_MANAGER_COMMAND table). If so, skip automatic power-on. + If the drain admitted anything, or an operation is still in flight, + hand off to the action loop; steps 3 to 6 do not run. + 3. After the delay, check if Rack Manager issued POWER_ON, POWER_OFF, + GRACEFUL_SHUT, or GRACEFUL_RESTART (CMD_STATUS_IN_PROGRESS or DONE + in RACK_MANAGER_COMMAND). If so, skip automatic power-on. 4. Check for CRITICAL leaks/alerts. 5. If none found and Switch-Host is OFFLINE, power it ON. 6. Update HOST_STATE. @@ -1278,22 +2212,29 @@ class BmcctldDaemon(daemon_base.DaemonBase): # received during the boot delay are processed in real time. Use a # monotonic clock so an NTP step during boot can't extend the wait. deadline = time.monotonic() + boot_delay + drain_handled_action = False while time.monotonic() < deadline: if self.stop_event.is_set(): return - try: - item = self.action_queue.get(timeout=1.0) - self._execute_action_item(item) - except queue.Empty: - continue + drain_handled_action = ( + self.operation_runner.process_next(timeout=1.0) or + drain_handled_action + ) - # After the boot delay, check whether a Rack Manager POWER_ON or POWER_OFF - # command was executed. We detect this by scanning the RACK_MANAGER_COMMAND - # table for any entry whose command is POWER_ON/POWER_OFF and whose status - # is IN_PROGRESS or DONE (set by the event handler and on_complete callback). + if drain_handled_action or self.operation_runner.current is not None: + self.log_notice( + "STARTUP: boot-delay drain handled an operation; " + "handing off to action loop") + return + + # After the boot delay, scan for Rack Manager commands whose power outcome + # already owns startup. IN_PROGRESS is set before enqueue and DONE by the + # completion callback. if self._rack_mgr_power_cmd_executed(): - self.log_notice("STARTUP: Rack Manager POWER_ON/POWER_OFF command executed " - "during boot delay; skipping automatic power-on") + self.log_notice( + "STARTUP: Rack Manager POWER_ON/POWER_OFF/GRACEFUL_SHUT/" + "GRACEFUL_RESTART command already IN_PROGRESS/DONE; " + "skipping automatic power-on") return if self.critical_event_checker.has_any_critical_event(): @@ -1304,71 +2245,25 @@ class BmcctldDaemon(daemon_base.DaemonBase): self.log_info("STARTUP: Switch-Host oper_status={}".format(oper_status)) if str(oper_status).upper() != SWITCH_HOST_ONLINE: - self._event_log.log_notice("STARTUP: Switch-Host OFFLINE — issuing power_on") - self.controller.power_on() + self._event_log.log_notice("STARTUP: Switch-Host OFFLINE — enqueueing power_on") + self.operation_runner.enqueue(ActionItem( + ACTION_POWER_ON, + "STARTUP", + action_priority(ACTION_POWER_ON), + startup=True, + )) else: self._event_log.log_notice("STARTUP: Switch-Host already ONLINE; refreshing HOST_STATE") self.controller.init_host_state() - def _execute_action_item(self, item): - """ - Execute a single ActionItem on the main thread. - Calls item.on_complete(success) when finished if a callback was provided. - """ - self._event_log.log_notice("Executing action '{}' (event: '{}')".format(item.action, item.event_desc)) - - current_status = self.controller.get_oper_status() - db_power_state = self.controller.get_db_power_state() - if item.action in (ACTION_GRACEFUL_SHUTDOWN, ACTION_POWER_OFF): - if current_status == SWITCH_HOST_OFFLINE or db_power_state in ( - SWITCH_HOST_POWERING_OFF, SWITCH_HOST_GRACEFUL_SHUTTING_DOWN): - self._event_log.log_notice( - "ACTION SKIPPED: '{}' trigger='{}' — " - "host already OFFLINE or shutting down (db_power_state={})".format( - item.action, item.event_desc, db_power_state)) - if item.on_complete: - item.on_complete(True) - return - elif item.action == ACTION_POWER_ON: - if current_status == SWITCH_HOST_ONLINE or db_power_state == SWITCH_HOST_POWERING_ON: - self._event_log.log_notice( - "ACTION SKIPPED: '{}' trigger='{}' — " - "host already ONLINE or POWERING_ON (db_power_state={})".format( - item.action, item.event_desc, db_power_state)) - if item.on_complete: - item.on_complete(True) - return - - success = False - if item.action == ACTION_GRACEFUL_SHUTDOWN: - success = self.graceful_shutdown.execute() - elif item.action == ACTION_POWER_OFF: - success = self.controller.power_off() - elif item.action == ACTION_POWER_ON: - success = self.controller.power_on() - elif item.action == ACTION_POWER_CYCLE: - success = self.controller.power_cycle() - else: - self.log_warning("Unknown action '{}' in queue; skipping".format(item.action)) - return - self._event_log.log_notice("ACTION DONE: action='{}' trigger='{}' " - "success={}".format(item.action, item.event_desc, success)) - if item.on_complete: - item.on_complete(success) - def _run_action_loop(self): """ - Main thread action processor. Dequeues and executes ActionItems - sequentially until stop_event is set. Actions are serialised naturally - by this single consumer — no extra locking needed. + Main-thread admission loop. The runner owns at most one operation + worker and processes queue conflicts until stop_event is set. """ self.log_info("Entering action processing loop") while not self.stop_event.is_set(): - try: - item = self.action_queue.get(timeout=1.0) - except queue.Empty: - continue - self._execute_action_item(item) + self.operation_runner.process_next(timeout=1.0) def run(self): """ @@ -1377,7 +2272,7 @@ class BmcctldDaemon(daemon_base.DaemonBase): 1. Sync HOST_STATE from live oper_status and seed CONFIG dedup state. 2. Start the BmcEventHandler thread (CLI admin up/down, Rack Manager, leaks). 3. If Switch-Host is already ONLINE: skip boot power-on sequence. - 4. Non-liquid-cooled + OFFLINE + admin_status=up: power on immediately. + 4. Non-liquid-cooled + OFFLINE + admin_status=up: enqueue power-on. 5. Liquid-cooled + OFFLINE: run _initial_power_on_sequence (boot delay). 6. Enter the action processing loop. @@ -1386,7 +2281,6 @@ class BmcctldDaemon(daemon_base.DaemonBase): enqueue ACTION_POWER_ON ahead of action-loop readiness. The BmcEventHandler thread (step 2) is started before the boot sequence so CLI admin commands and other CONFIG/STATE events are captured regardless of the startup path. - startup power-on or the action loop. Returns False to terminate the outer while-loop. """ @@ -1405,39 +2299,51 @@ class BmcctldDaemon(daemon_base.DaemonBase): name="bmcctld-events", ) event_thread.start() + try: + if not self.event_handler.wait_for_event_subscriptions(): + return False - # If Switch-Host is already ONLINE (daemon restart), skip the boot - # sequence entirely and go straight to the action loop. - if self.controller.get_oper_status() == SWITCH_HOST_ONLINE: - self._event_log.log_notice("STARTUP: Switch-Host already ONLINE (daemon restart); " - "skipping boot sequence") - elif not self.chassis.is_liquid_cooled(): - admin_status = self.policy_reader.get_switch_host_admin_status() - if admin_status != ADMIN_UP: - self._event_log.log_notice( - "STARTUP: SWITCH-HOST admin_status='{}'; " - "Switch-Host will remain powered off. " - "Run 'config chassis modules startup SWITCH-HOST' to power on.".format( - admin_status)) + # If Switch-Host is already ONLINE (daemon restart), skip the boot + # sequence entirely and go straight to the action loop. + if self.controller.get_oper_status() == SWITCH_HOST_ONLINE: + self._event_log.log_notice("STARTUP: Switch-Host already ONLINE (daemon restart); " + "skipping boot sequence") + elif not self.chassis.is_liquid_cooled(): + admin_status = self.policy_reader.get_switch_host_admin_status() + if admin_status != ADMIN_UP: + self._event_log.log_notice( + "STARTUP: SWITCH-HOST admin_status='{}'; " + "Switch-Host will remain powered off. " + "Run 'config chassis modules startup SWITCH-HOST' to power on.".format( + admin_status)) + else: + self._event_log.log_notice("STARTUP: non-liquid-cooled system — " + "enqueueing power_on, no additional checks") + self.operation_runner.enqueue(ActionItem( + ACTION_POWER_ON, + "STARTUP", + action_priority(ACTION_POWER_ON), + startup=True, + )) else: - self._event_log.log_notice("STARTUP: non-liquid-cooled system — " - "immediate power_on, no additional checks") - self.controller.power_on() - else: - # Liquid-cooled: wait for Rack Manager to verify liquid flow, check for - # leaks/alerts, then power on if safe. - self._initial_power_on_sequence() - if self.stop_event.is_set(): - event_thread.join(timeout=5) - return False + # Liquid-cooled: wait for Rack Manager to verify liquid flow, check for + # leaks/alerts, then power on if safe. + self._initial_power_on_sequence() + if self.stop_event.is_set(): + return False - self._run_action_loop() - event_thread.join(timeout=5) - return False + self._run_action_loop() + return False + finally: + try: + self.deinit() + finally: + event_thread.join(timeout=5) def deinit(self): self.log_notice("bmcctld shutting down") self.stop_event.set() + self.operation_runner.stop() # ============================================================ @@ -1452,7 +2358,6 @@ def main(): pass daemon.log_info("Shutting down with exit code {}...".format(exit_code)) - daemon.deinit() return exit_code diff --git a/sonic-bmcctld/tests/mock_swsscommon.py b/sonic-bmcctld/tests/mock_swsscommon.py index 36fda0cf1..e7ea25845 100644 --- a/sonic-bmcctld/tests/mock_swsscommon.py +++ b/sonic-bmcctld/tests/mock_swsscommon.py @@ -22,5 +22,5 @@ RedisPipeline, STATE_DB, CONFIG_DB, + reset_mock_db, ) - diff --git a/sonic-bmcctld/tests/mocked_libs/grpc.py b/sonic-bmcctld/tests/mocked_libs/grpc.py new file mode 100644 index 000000000..3873397c3 --- /dev/null +++ b/sonic-bmcctld/tests/mocked_libs/grpc.py @@ -0,0 +1,5 @@ +"""Minimal gRPC import surface; transport tests inject their own client API.""" + + +class RpcError(Exception): + pass diff --git a/sonic-bmcctld/tests/mocked_libs/sonic_grpc/__init__.py b/sonic-bmcctld/tests/mocked_libs/sonic_grpc/__init__.py new file mode 100644 index 000000000..a4ebc7557 --- /dev/null +++ b/sonic-bmcctld/tests/mocked_libs/sonic_grpc/__init__.py @@ -0,0 +1 @@ +"""gNOI import stubs for isolated daemon unit tests.""" diff --git a/sonic-bmcctld/tests/mocked_libs/sonic_grpc/gnoi/__init__.py b/sonic-bmcctld/tests/mocked_libs/sonic_grpc/gnoi/__init__.py new file mode 100644 index 000000000..a4ebc7557 --- /dev/null +++ b/sonic-bmcctld/tests/mocked_libs/sonic_grpc/gnoi/__init__.py @@ -0,0 +1 @@ +"""gNOI import stubs for isolated daemon unit tests.""" diff --git a/sonic-bmcctld/tests/mocked_libs/sonic_grpc/gnoi/client.py b/sonic-bmcctld/tests/mocked_libs/sonic_grpc/gnoi/client.py new file mode 100644 index 000000000..56e0e31dc --- /dev/null +++ b/sonic-bmcctld/tests/mocked_libs/sonic_grpc/gnoi/client.py @@ -0,0 +1,6 @@ +"""Placeholder; requester tests must inject the client they exercise.""" + + +class GnoiClient: + def __init__(self, *args, **kwargs): + raise AssertionError("GnoiClient must be mocked by the unit test") diff --git a/sonic-bmcctld/tests/mocked_libs/sonic_grpc/gnoi/system_pb2.py b/sonic-bmcctld/tests/mocked_libs/sonic_grpc/gnoi/system_pb2.py new file mode 100644 index 000000000..f7fb8d26b --- /dev/null +++ b/sonic-bmcctld/tests/mocked_libs/sonic_grpc/gnoi/system_pb2.py @@ -0,0 +1,7 @@ +"""Protocol constants used by report-classification unit tests.""" + +HALT = 3 + + +class RebootStatus: + STATUS_SUCCESS = 1 diff --git a/sonic-bmcctld/tests/mocked_libs/sonic_py_common/device_info.py b/sonic-bmcctld/tests/mocked_libs/sonic_py_common/device_info.py new file mode 100644 index 000000000..efb2ad7f5 --- /dev/null +++ b/sonic-bmcctld/tests/mocked_libs/sonic_py_common/device_info.py @@ -0,0 +1,13 @@ +"""Minimal device_info surface used by bmcctld unit tests.""" + + +def is_switch_bmc(): + return False + + +def get_platform_json_data(): + return None + + +def get_path_to_platform_dir(): + return "" diff --git a/sonic-bmcctld/tests/mocked_libs/swsscommon/swsscommon.py b/sonic-bmcctld/tests/mocked_libs/swsscommon/swsscommon.py index 70d80753e..6a7d32db3 100644 --- a/sonic-bmcctld/tests/mocked_libs/swsscommon/swsscommon.py +++ b/sonic-bmcctld/tests/mocked_libs/swsscommon/swsscommon.py @@ -43,12 +43,22 @@ def isInit(): STATE_DB = '' CONFIG_DB = '' +_MOCK_DATABASE = {} + + +def reset_mock_db(): + _MOCK_DATABASE.clear() + class Table: def __init__(self, *argv): self.db_or_pipe = argv[0] if argv else None self.table_name = argv[1] if len(argv) > 1 else '' - self.mock_dict = {} + if self.db_or_pipe is None: + self.mock_dict = {} + else: + self.mock_dict = _MOCK_DATABASE.setdefault( + (self.db_or_pipe, self.table_name), {}) def _del(self, key): if key in self.mock_dict: diff --git a/sonic-bmcctld/tests/test_bmcctld.py b/sonic-bmcctld/tests/test_bmcctld.py index 640b52b97..4b545161e 100644 --- a/sonic-bmcctld/tests/test_bmcctld.py +++ b/sonic-bmcctld/tests/test_bmcctld.py @@ -6,10 +6,14 @@ import os import sys +import itertools +import queue import threading import time import importlib.util import importlib.machinery +import builtins +from types import SimpleNamespace def load_source(module_name, module_path): loader = importlib.machinery.SourceFileLoader(module_name, module_path) @@ -50,7 +54,7 @@ def load_source(module_name, module_path): os.environ["BMCCTLD_UNIT_TESTING"] = "1" from sonic_py_common import daemon_base # noqa: E402 -daemon_base.db_connect = MagicMock() +daemon_base.db_connect = MagicMock(side_effect=lambda db_name: db_name) load_source('bmcctld', os.path.join(scripts_path, 'bmcctld')) import bmcctld # noqa: E402 (loaded via load_source above) @@ -58,11 +62,58 @@ def load_source(module_name, module_path): from .mock_platform import MockChassis, MockModule from .mock_swsscommon import Table, FieldValuePairs +TEST_REQUEST_ID = "12345678-1234-1234-1234-123456789abc" +TEST_UUID1 = "3f2b1c8a-1234-1abc-8def-0123456789ab" +TEST_UUID4 = "3f2b1c8a-1234-4abc-8def-0123456789ab" + + +@pytest.mark.parametrize("dependency", [ + "grpc", "sonic_grpc.gnoi.client", "sonic_grpc.gnoi", +]) +def test_missing_required_gnoi_import_fails_loading(dependency, monkeypatch): + original_import = builtins.__import__ + + def import_without_dependency(name, *args, **kwargs): + if name == dependency: + raise ImportError("missing required dependency: " + name) + return original_import(name, *args, **kwargs) + + module_name = "bmcctld_missing_dependency" + loader = importlib.machinery.SourceFileLoader( + module_name, os.path.join(scripts_path, "bmcctld")) + spec = importlib.util.spec_from_loader(module_name, loader) + monkeypatch.setitem(sys.modules, module_name, importlib.util.module_from_spec(spec)) + with patch("builtins.__import__", side_effect=import_without_dependency): + with pytest.raises(ImportError, match="missing required dependency"): + load_source(module_name, os.path.join(scripts_path, "bmcctld")) + + +def _make_operation(action, event_desc="test", priority=None, callback=None, + rack_cmd_key=None): + if priority is None: + priority = bmcctld.action_priority(action) + item = bmcctld.ActionItem( + action, event_desc, priority, + on_complete=callback, rack_cmd_key=rack_cmd_key) + return bmcctld.Operation( + item, TEST_REQUEST_ID, + bmcctld.OperationRunner.INITIAL_STAGES[action]) + + +def _dequeue_item(action_queue): + return action_queue.get_nowait()[2] + # -------------------------------------------------------------------------- # Fixtures # -------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def reset_database_mocks(): + bmcctld.swsscommon.reset_mock_db() + daemon_base.db_connect.reset_mock() + + @pytest.fixture(autouse=True) def silence_logs(monkeypatch): """Suppress all syslog calls during tests.""" @@ -88,10 +139,7 @@ def chassis(): @pytest.fixture def controller(chassis): - ctrl = bmcctld.SwitchHostController(chassis) - ctrl.host_state_table = Table(None, bmcctld.HOST_STATE_TABLE) - ctrl.chassis_module_info_table = Table(None, bmcctld.CHASSIS_MODULE_INFO_TABLE) - return ctrl + return bmcctld.SwitchHostController(chassis) @pytest.fixture @@ -103,8 +151,6 @@ def policy_reader(): @pytest.fixture def critical_event_checker(policy_reader): lc = bmcctld.CriticalEventChecker(policy_reader) - lc._system_leak_table = Table(None, bmcctld.SYSTEM_LEAK_STATUS_TABLE) - lc._rack_alert_table = Table(None, bmcctld.RACK_MANAGER_ALERT_TABLE) return lc @@ -118,9 +164,11 @@ def graceful_shutdown(controller, policy_reader): def event_handler(controller, policy_reader, critical_event_checker): stop_event = threading.Event() stop_event.set() # Prevent blocking in tests - action_queue = __import__('queue').Queue() - eh = bmcctld.BmcEventHandler(action_queue, policy_reader, critical_event_checker, stop_event, - controller) + action_queue = queue.PriorityQueue() + action_sequence = itertools.count() + eh = bmcctld.BmcEventHandler( + action_queue, action_sequence, policy_reader, + critical_event_checker, stop_event, controller) # Replace live DB tables with in-memory mocks eh._cmd_table = Table(None, bmcctld.RACK_MANAGER_COMMAND_TABLE) return eh @@ -150,23 +198,23 @@ def test_get_oper_status_offline(self, chassis, controller): def test_power_on_calls_set_admin_state(self, chassis, controller): chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) - result = controller.power_on() - assert result is True + result = controller.power_on(_make_operation(bmcctld.ACTION_POWER_ON)) + assert result == bmcctld.PowerCallResult.CONFIRMED assert chassis.switch_host.get_admin_state() is True def test_power_off_calls_set_admin_state(self, chassis, controller): chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) - result = controller.power_off() - assert result is True + result = controller.power_off(_make_operation(bmcctld.ACTION_POWER_OFF)) + assert result == bmcctld.PowerCallResult.CONFIRMED assert chassis.switch_host.get_admin_state() is False def test_power_cycle_calls_do_power_cycle(self, chassis, controller): - result = controller.power_cycle() - assert result is True + result = controller.power_cycle(_make_operation(bmcctld.ACTION_POWER_CYCLE)) + assert result == bmcctld.PowerCallResult.CONFIRMED assert chassis.switch_host.power_cycle_called is True def test_power_on_updates_host_state(self, chassis, controller): - controller.power_on() + controller.power_on(_make_operation(bmcctld.ACTION_POWER_ON)) result = controller.host_state_table.get(bmcctld.HOST_STATE_KEY) assert result[0] is True state = dict(result[1]) @@ -175,7 +223,7 @@ def test_power_on_updates_host_state(self, chassis, controller): def test_power_off_updates_host_state(self, chassis, controller): chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) - controller.power_off() + controller.power_off(_make_operation(bmcctld.ACTION_POWER_OFF)) result = controller.host_state_table.get(bmcctld.HOST_STATE_KEY) assert result[0] is True state = dict(result[1]) @@ -192,7 +240,7 @@ def interceptor(up): captured.update(dict(result[1])) original(up) chassis.switch_host.set_admin_state = interceptor - controller.power_on() + controller.power_on(_make_operation(bmcctld.ACTION_POWER_ON)) assert captured.get(bmcctld.FIELD_DEVICE_POWER_STATE) == bmcctld.SWITCH_HOST_POWERING_ON assert captured.get(bmcctld.FIELD_DEVICE_STATUS) in (bmcctld.SWITCH_HOST_ONLINE, bmcctld.SWITCH_HOST_OFFLINE) # Final entry must reflect POWER_ON and confirmed ONLINE @@ -211,7 +259,7 @@ def interceptor(up): captured.update(dict(result[1])) original(up) chassis.switch_host.set_admin_state = interceptor - controller.power_off() + controller.power_off(_make_operation(bmcctld.ACTION_POWER_OFF)) assert captured.get(bmcctld.FIELD_DEVICE_POWER_STATE) == bmcctld.SWITCH_HOST_POWERING_OFF assert captured.get(bmcctld.FIELD_DEVICE_STATUS) in (bmcctld.SWITCH_HOST_ONLINE, bmcctld.SWITCH_HOST_OFFLINE) # Final entry must reflect POWER_OFF and confirmed OFFLINE @@ -229,7 +277,7 @@ def interceptor(): captured.update(dict(result[1])) original() chassis.switch_host.do_power_cycle = interceptor - controller.power_cycle() + controller.power_cycle(_make_operation(bmcctld.ACTION_POWER_CYCLE)) assert captured.get(bmcctld.FIELD_DEVICE_POWER_STATE) == bmcctld.SWITCH_HOST_POWER_CYCLING assert captured.get(bmcctld.FIELD_DEVICE_STATUS) in (bmcctld.SWITCH_HOST_ONLINE, bmcctld.SWITCH_HOST_OFFLINE) # Final entry must reflect POWER_CYCLE and confirmed ONLINE @@ -239,30 +287,51 @@ def interceptor(): def test_get_db_power_state(self, chassis, controller): """get_db_power_state returns the value stored by the last _update_host_state call.""" - controller.power_on() + controller.power_on(_make_operation(bmcctld.ACTION_POWER_ON)) assert controller.get_db_power_state() == bmcctld.POWER_STATE_ON - def test_power_on_rolls_back_state_on_exception(self, chassis, controller): - """If set_admin_state raises, STATE_DB is restored to the pre-call snapshot.""" + def test_power_on_leaves_transition_on_issued_call_exception(self, chassis, controller): + """If set_admin_state raises, the raise may have taken effect: POWERING_ON stays.""" chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) - controller.power_off() # seed a known prior state (OFFLINE / POWER_OFF) + controller.power_off(_make_operation(bmcctld.ACTION_POWER_OFF)) chassis.switch_host.set_admin_state = MagicMock(side_effect=RuntimeError("hw fault")) - result = controller.power_on() - assert result is False + result = controller.power_on(_make_operation(bmcctld.ACTION_POWER_ON)) + assert result == bmcctld.PowerCallResult.NOT_CONFIRMED state = dict(controller.host_state_table.get(bmcctld.HOST_STATE_KEY)[1]) assert state[bmcctld.FIELD_DEVICE_STATUS] == bmcctld.SWITCH_HOST_OFFLINE + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.SWITCH_HOST_POWERING_ON + + def test_power_on_restores_state_when_raise_was_never_issued(self, chassis, controller): + """An exception before the platform call restores the pre-call snapshot.""" + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + controller.power_off(_make_operation(bmcctld.ACTION_POWER_OFF)) + chassis.switch_host.set_admin_state = MagicMock() + real_update = controller._update_host_state + writes = [] + + def failing_transitional_write(power_state, device_status=None): + writes.append(power_state) + if power_state == bmcctld.SWITCH_HOST_POWERING_ON: + raise RuntimeError("write failed") + return real_update(power_state, device_status) + + controller._update_host_state = failing_transitional_write + result = controller.power_on(_make_operation(bmcctld.ACTION_POWER_ON)) + assert result == bmcctld.PowerCallResult.NOT_CONFIRMED + chassis.switch_host.set_admin_state.assert_not_called() + assert writes == [bmcctld.SWITCH_HOST_POWERING_ON, bmcctld.POWER_STATE_OFF] + state = dict(controller.host_state_table.get(bmcctld.HOST_STATE_KEY)[1]) assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF - def test_power_off_rolls_back_state_on_exception(self, chassis, controller): - """If set_admin_state raises, STATE_DB is restored to the pre-call snapshot.""" + def test_power_off_leaves_transition_on_exception(self, chassis, controller): chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) - controller.power_on() # seed a known prior state (ONLINE / POWER_ON) + controller.power_on(_make_operation(bmcctld.ACTION_POWER_ON)) chassis.switch_host.set_admin_state = MagicMock(side_effect=RuntimeError("hw fault")) - result = controller.power_off() - assert result is False + result = controller.power_off(_make_operation(bmcctld.ACTION_POWER_OFF)) + assert result == bmcctld.PowerCallResult.NOT_CONFIRMED state = dict(controller.host_state_table.get(bmcctld.HOST_STATE_KEY)[1]) assert state[bmcctld.FIELD_DEVICE_STATUS] == bmcctld.SWITCH_HOST_ONLINE - assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_ON + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.SWITCH_HOST_POWERING_OFF def test_get_switch_host_module_by_type(self, chassis): """If a module explicitly returns MODULE_TYPE_SWITCH_HOST it is selected.""" @@ -298,13 +367,144 @@ def test_init_host_state_infers_power_off_when_not_available(self, chassis, cont assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF assert state[bmcctld.FIELD_DEVICE_STATUS] == bmcctld.SWITCH_HOST_OFFLINE + @pytest.mark.parametrize("stale_state", [ + bmcctld.SWITCH_HOST_POWERING_ON, + bmcctld.SWITCH_HOST_POWERING_OFF, + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN, + bmcctld.SWITCH_HOST_POWER_CYCLING, + bmcctld.POWER_STATE_ON, + bmcctld.POWER_STATE_CYCLE, + ]) + def test_init_host_state_keeps_label_other_than_power_off_while_offline( + self, chassis, controller, stale_state): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + controller._update_host_state(stale_state) + controller.init_host_state() + state = dict(controller.host_state_table.get(bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == stale_state + assert state[bmcctld.FIELD_DEVICE_STATUS] == bmcctld.SWITCH_HOST_OFFLINE + + def test_init_host_state_resolves_transitional_label_when_online( + self, chassis, controller): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + controller._update_host_state(bmcctld.SWITCH_HOST_POWERING_ON) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + controller.init_host_state() + state = dict(controller.host_state_table.get(bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_ON + + @pytest.mark.parametrize( + "death_point,stale_state,live_status,op_result,stale_reason", [ + ("before_host_accept", bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN, + MockModule.MODULE_STATUS_ONLINE, "-", "-"), + ("after_accept_before_watchdog", bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN, + MockModule.MODULE_STATUS_ONLINE, "-", "-"), + ("after_watchdog_arm", bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN, + MockModule.MODULE_STATUS_ONLINE, "-", "-"), + ("before_transitional_write", bmcctld.POWER_STATE_ON, + MockModule.MODULE_STATUS_ONLINE, None, + bmcctld.OP_REASON_UNCLASSIFIED), + ("before_terminal_record", bmcctld.SWITCH_HOST_POWERING_OFF, + MockModule.MODULE_STATUS_OFFLINE, "-", "-"), + ]) + @pytest.mark.parametrize("request_id", [TEST_UUID1, TEST_UUID4, TEST_REQUEST_ID]) + def test_dth_stale_operation_is_abandoned_on_startup( + self, chassis, death_point, stale_state, live_status, op_result, + stale_reason, request_id): + retained_state = Table("STATE_DB", bmcctld.HOST_STATE_TABLE) + fields = [ + (bmcctld.FIELD_DEVICE_POWER_STATE, stale_state), + (bmcctld.FIELD_DEVICE_STATUS, bmcctld.SWITCH_HOST_ONLINE), + (bmcctld.FIELD_OP_REQUEST_ID, request_id), + (bmcctld.FIELD_OP_TRIGGER, death_point), + (bmcctld.FIELD_OP_REASON, stale_reason), + ] + if op_result is not None: + fields.append((bmcctld.FIELD_OP_RESULT, op_result)) + retained_state.set( + bmcctld.HOST_STATE_KEY, FieldValuePairs(fields)) + + chassis.switch_host.set_oper_status(live_status) + chassis.switch_host.set_admin_state = MagicMock() + chassis.switch_host.do_power_cycle = MagicMock() + chassis.set_liquid_cooled(False) + with patch('sonic_platform.platform.Platform') as mock_platform: + mock_platform.return_value.get_chassis.return_value = chassis + fresh_daemon = bmcctld.BmcctldDaemon( + bmcctld.SYSLOG_IDENTIFIER) + fresh_daemon.policy_reader.get_switch_host_admin_status = MagicMock( + return_value=bmcctld.ADMIN_DOWN) + fresh_daemon.event_handler.run_event_loop = MagicMock( + side_effect=fresh_daemon.event_handler._subscription_ready.set) + fresh_daemon._run_action_loop = MagicMock() + + assert fresh_daemon.run() is False + + state = dict(retained_state.get(bmcctld.HOST_STATE_KEY)[1]) + if live_status == MockModule.MODULE_STATUS_ONLINE: + expected_power_state = bmcctld.POWER_STATE_ON + elif stale_state != bmcctld.POWER_STATE_OFF: + expected_power_state = stale_state + else: + expected_power_state = bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == expected_power_state + assert state[bmcctld.FIELD_DEVICE_STATUS] == str(live_status).upper() + assert state[bmcctld.FIELD_OP_REQUEST_ID] == request_id + assert state[bmcctld.FIELD_OP_TRIGGER] == death_point + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_ABANDONED + assert state[bmcctld.FIELD_OP_REASON] == "-" + fresh_daemon.controller.log_warning.assert_called_once_with( + "OP_ABANDONED request_id={} stale_state={}".format( + request_id, stale_state)) + assert fresh_daemon.operation_runner.current is None + assert fresh_daemon.action_queue.empty() + chassis.switch_host.set_admin_state.assert_not_called() + chassis.switch_host.do_power_cycle.assert_not_called() + + @pytest.mark.parametrize("request_id", [ + "-", + "", + TEST_UUID4.upper(), + "not-a-uuid", + ]) + def test_dth_invalid_request_id_is_not_abandoned( + self, chassis, controller, request_id): + controller.host_state_table.set( + bmcctld.HOST_STATE_KEY, FieldValuePairs([ + (bmcctld.FIELD_DEVICE_POWER_STATE, + bmcctld.SWITCH_HOST_POWERING_OFF), + (bmcctld.FIELD_OP_REQUEST_ID, request_id), + (bmcctld.FIELD_OP_RESULT, "-"), + ])) + controller.init_host_state() + state = dict(controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == "-" + controller.log_warning.assert_not_called() + + def test_dth_terminal_operation_is_not_rewritten(self, controller): + controller.host_state_table.set( + bmcctld.HOST_STATE_KEY, FieldValuePairs([ + (bmcctld.FIELD_DEVICE_POWER_STATE, + bmcctld.SWITCH_HOST_POWERING_OFF), + (bmcctld.FIELD_OP_REQUEST_ID, TEST_UUID4), + (bmcctld.FIELD_OP_RESULT, bmcctld.OP_RESULT_POWER_OFF_FAILED), + (bmcctld.FIELD_OP_REASON, "-"), + ])) + controller.init_host_state() + state = dict(controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_OFF_FAILED + controller.log_warning.assert_not_called() + # -- _verify_oper_status tests -- def test_verify_oper_status_matches_immediately(self, chassis, controller): chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) with patch('time.sleep') as mock_sleep: result = controller._verify_oper_status(bmcctld.SWITCH_HOST_ONLINE, 30, "test") - assert result is True + assert result == bmcctld.PowerCallResult.CONFIRMED mock_sleep.assert_not_called() @patch('time.sleep') @@ -315,36 +515,210 @@ def test_verify_oper_status_timeout(self, mock_monotonic, mock_sleep, chassis, c mock_monotonic.side_effect = [0, 0, 31, 31] chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) result = controller._verify_oper_status(bmcctld.SWITCH_HOST_ONLINE, 30, "test") - assert result is False + assert result == bmcctld.PowerCallResult.NOT_CONFIRMED mock_sleep.assert_called_once_with(bmcctld.POWER_VERIFY_POLL_INTERVAL_SECS) def test_power_on_returns_false_when_verify_fails(self, chassis, controller): - with patch.object(controller, '_verify_oper_status', return_value=False): - result = controller.power_on() - assert result is False + with patch.object( + controller, '_verify_oper_status', + return_value=bmcctld.PowerCallResult.NOT_CONFIRMED): + result = controller.power_on(_make_operation(bmcctld.ACTION_POWER_ON)) + assert result == bmcctld.PowerCallResult.NOT_CONFIRMED assert chassis.switch_host.get_admin_state() is True # API was still called def test_power_off_returns_false_when_verify_fails(self, chassis, controller): - with patch.object(controller, '_verify_oper_status', return_value=False): - result = controller.power_off() - assert result is False + with patch.object( + controller, '_verify_oper_status', + return_value=bmcctld.PowerCallResult.NOT_CONFIRMED): + result = controller.power_off(_make_operation(bmcctld.ACTION_POWER_OFF)) + assert result == bmcctld.PowerCallResult.NOT_CONFIRMED assert chassis.switch_host.get_admin_state() is False # API was still called def test_power_cycle_returns_false_when_verify_fails(self, chassis, controller): - with patch.object(controller, '_verify_oper_status', return_value=False): - result = controller.power_cycle() - assert result is False + with patch.object( + controller, '_verify_oper_status', + return_value=bmcctld.PowerCallResult.NOT_CONFIRMED): + result = controller.power_cycle(_make_operation(bmcctld.ACTION_POWER_CYCLE)) + assert result == bmcctld.PowerCallResult.NOT_CONFIRMED assert chassis.switch_host.power_cycle_called is True # API was still called def test_power_cycle_uses_double_timeout(self, chassis, controller): - with patch.object(controller, '_verify_oper_status', return_value=True) as mock_verify: - controller.power_cycle() + with patch.object( + controller, '_verify_oper_status', + return_value=bmcctld.PowerCallResult.CONFIRMED) as mock_verify: + operation = _make_operation(bmcctld.ACTION_POWER_CYCLE) + controller.power_cycle(operation) mock_verify.assert_called_once_with( bmcctld.SWITCH_HOST_ONLINE, bmcctld.POWER_VERIFY_TIMEOUT_SECS * 2, - "power_cycle" + "power_cycle", + None, ) + @pytest.mark.parametrize( + "action, method_name", + [ + (bmcctld.ACTION_POWER_OFF, "power_off"), + (bmcctld.ACTION_POWER_ON, "power_on"), + (bmcctld.ACTION_POWER_CYCLE, "power_cycle"), + ], + ) + def test_p_cancel_before_wrapper_entry_issues_no_platform_call( + self, action, method_name, chassis, controller): + operation = _make_operation(action) + operation.cancel.set() + chassis.switch_host.set_admin_state = MagicMock() + chassis.switch_host.do_power_cycle = MagicMock() + result = getattr(controller, method_name)(operation, operation.cancel) + assert result == bmcctld.PowerCallResult.CANCELLED + chassis.switch_host.set_admin_state.assert_not_called() + chassis.switch_host.do_power_cycle.assert_not_called() + + @pytest.mark.parametrize( + "action, method_name", + [ + (bmcctld.ACTION_POWER_OFF, "power_off"), + (bmcctld.ACTION_POWER_ON, "power_on"), + (bmcctld.ACTION_POWER_CYCLE, "power_cycle"), + ], + ) + def test_p_cancel_under_lock_issues_no_platform_call( + self, action, method_name, chassis, controller): + operation = _make_operation(action) + chassis.switch_host.set_admin_state = MagicMock() + chassis.switch_host.do_power_cycle = MagicMock() + + class CancelOnEnter: + def __enter__(self): + operation.cancel.set() + def __exit__(self, exc_type, exc_value, tb): + return False + + controller._power_lock = CancelOnEnter() + result = getattr(controller, method_name)(operation, operation.cancel) + assert result == bmcctld.PowerCallResult.CANCELLED + chassis.switch_host.set_admin_state.assert_not_called() + chassis.switch_host.do_power_cycle.assert_not_called() + + @pytest.mark.parametrize( + "action, method_name", + [ + (bmcctld.ACTION_POWER_ON, "power_on"), + (bmcctld.ACTION_POWER_CYCLE, "power_cycle"), + ], + ) + def test_s_raise_is_refused_on_under_lock_critical_reread( + self, action, method_name, chassis, controller): + checker = MagicMock() + checker.has_any_critical_event.return_value = True + controller.set_critical_event_checker(checker) + chassis.switch_host.set_admin_state = MagicMock() + chassis.switch_host.do_power_cycle = MagicMock() + operation = _make_operation(action) + result = getattr(controller, method_name)(operation, operation.cancel) + assert result == bmcctld.PowerCallResult.REFUSED_LEAK + checker.has_any_critical_event.assert_called_once() + chassis.switch_host.set_admin_state.assert_not_called() + chassis.switch_host.do_power_cycle.assert_not_called() + + def test_p_verify_wait_is_cancelled_without_poll_interval_delay( + self, chassis, controller): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + cancel = MagicMock() + cancel.is_set.return_value = False + cancel.wait.return_value = True + with patch('bmcctld.time.sleep') as sleep: + result = controller._verify_oper_status( + bmcctld.SWITCH_HOST_ONLINE, 60, "cancel-test", cancel) + assert result == bmcctld.PowerCallResult.CANCELLED + cancel.wait.assert_called_once() + sleep.assert_not_called() + + @pytest.mark.parametrize( + "action, method_name, platform_method", + [ + (bmcctld.ACTION_POWER_OFF, "power_off", "set_admin_state"), + (bmcctld.ACTION_POWER_ON, "power_on", "set_admin_state"), + (bmcctld.ACTION_POWER_CYCLE, "power_cycle", "do_power_cycle"), + ], + ) + def test_p_cancel_during_issued_call_wins_over_call_exception( + self, action, method_name, platform_method, chassis, controller): + operation = _make_operation(action) + + def cancel_then_raise(*_args): + operation.cancel.set() + raise RuntimeError("platform call failed after cancellation") + + setattr(chassis.switch_host, platform_method, cancel_then_raise) + result = getattr(controller, method_name)(operation, operation.cancel) + + assert result == bmcctld.PowerCallResult.CANCELLED + + @pytest.mark.parametrize( + "action, method_name, issued_stage, confirmed_stage", + [ + (bmcctld.ACTION_POWER_OFF, "power_off", + bmcctld.STAGE_POWER_OFF_ISSUED, + bmcctld.STAGE_POWER_OFF_CONFIRMED), + (bmcctld.ACTION_POWER_ON, "power_on", + bmcctld.STAGE_POWER_ON_ISSUED, + bmcctld.STAGE_POWER_ON_CONFIRMED), + (bmcctld.ACTION_POWER_CYCLE, "power_cycle", + bmcctld.STAGE_POWER_CYCLE_ISSUED, + bmcctld.STAGE_POWER_CYCLE_CONFIRMED), + ], + ) + def test_q_stage_is_issued_during_call_and_confirmed_after_verify( + self, action, method_name, issued_stage, confirmed_stage, + chassis, controller): + operation = _make_operation(action) + observed = [] + if action == bmcctld.ACTION_POWER_CYCLE: + original = chassis.switch_host.do_power_cycle + def platform_call(): + observed.append(operation.stage) + original() + chassis.switch_host.do_power_cycle = platform_call + else: + original = chassis.switch_host.set_admin_state + def platform_call(up): + observed.append(operation.stage) + original(up) + chassis.switch_host.set_admin_state = platform_call + if action == bmcctld.ACTION_POWER_OFF: + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + result = getattr(controller, method_name)(operation, operation.cancel) + assert result == bmcctld.PowerCallResult.CONFIRMED + assert observed == [issued_stage] + assert operation.stage == confirmed_stage + + def test_w_power_on_timeout_leaves_transition(self, chassis, controller): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + operation = _make_operation(bmcctld.ACTION_POWER_ON) + with patch.object( + controller, '_verify_oper_status', + return_value=bmcctld.PowerCallResult.NOT_CONFIRMED): + result = controller.power_on(operation, operation.cancel) + assert result == bmcctld.PowerCallResult.NOT_CONFIRMED + state = dict(controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_ON + assert state[bmcctld.FIELD_DEVICE_STATUS] == bmcctld.SWITCH_HOST_ONLINE + + def test_w_power_cycle_timeout_leaves_transition(self, chassis, controller): + operation = _make_operation(bmcctld.ACTION_POWER_CYCLE) + with patch.object( + controller, '_verify_oper_status', + return_value=bmcctld.PowerCallResult.NOT_CONFIRMED): + result = controller.power_cycle(operation, operation.cancel) + assert result == bmcctld.PowerCallResult.NOT_CONFIRMED + state = dict(controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWER_CYCLING + # -------------------------------------------------------------------------- # Tests: PolicyReader @@ -369,9 +743,25 @@ def test_get_power_on_delay_custom(self, policy_reader): assert policy_reader.get_power_on_delay() == 60 def test_get_graceful_shutdown_timeout_default(self, policy_reader): - """When no CHASSIS_MODULE|SWITCH-HOST entry exists, graceful_shutdown_timeout defaults to 0.""" + """A missing module row uses the graceful timeout default.""" with patch.object(bmcctld.swsscommon, 'Table', return_value=Table(None, "T")): - assert policy_reader.get_graceful_shutdown_timeout() == bmcctld.DEFAULT_SHUTDOWN_DELAY_SECS + assert policy_reader.get_graceful_shutdown_timeout() == 120 + + @pytest.mark.parametrize("fields", [ + {bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_UP}, + {bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT: "invalid"}, + {bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT: ""}, + {bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT: None}, + ], ids=["missing-field", "invalid", "empty", "null"]) + def test_get_graceful_shutdown_timeout_fallback(self, policy_reader, fields): + with patch.object(policy_reader, "_get_chassis_module_entry", return_value=fields): + assert policy_reader.get_graceful_shutdown_timeout() == 120 + + @pytest.mark.parametrize("seconds", [0, 30, 120, 300]) + def test_get_graceful_shutdown_timeout_preserves_explicit_value(self, policy_reader, seconds): + fields = {bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT: str(seconds)} + with patch.object(policy_reader, "_get_chassis_module_entry", return_value=fields): + assert policy_reader.get_graceful_shutdown_timeout() == seconds def test_get_graceful_shutdown_timeout_zero(self, policy_reader): tbl = Table(None, bmcctld.CHASSIS_MODULE_TABLE) @@ -379,6 +769,31 @@ def test_get_graceful_shutdown_timeout_zero(self, policy_reader): with patch.object(bmcctld.swsscommon, 'Table', return_value=tbl): assert policy_reader.get_graceful_shutdown_timeout() == 0 + @pytest.mark.parametrize("field", [None, "ca_crt", "client_crt", "client_key"]) + def test_certificate_defaults_and_partial_overrides(self, field, policy_reader): + table = policy_reader._thread_database.table("CONFIG_DB", "BMC_GNOI") + expected = { + "ca_crt": "/etc/sonic/bmc-link/ca.crt", + "client_crt": "/etc/sonic/bmc-link/client.crt", + "client_key": "/etc/sonic/bmc-link/client.key", + } + if field: + _set_table_entry(table, "certs", {field: "/etc/sonic/custom.pem"}) + expected[field] = "/etc/sonic/custom.pem" + with patch.object(table, "set") as write_row: + paths = policy_reader.get_gnoi_cert_paths() + assert paths == expected + paths["ca_crt"] = "modified local copy" + assert policy_reader.get_gnoi_cert_paths() == expected + write_row.assert_not_called() + + def test_certificate_reader_projects_only_known_fields(self, policy_reader): + table = policy_reader._thread_database.table("CONFIG_DB", "BMC_GNOI") + _set_table_entry(table, "certs", {"ca_crt": "", "extra": "not a path"}) + paths = policy_reader.get_gnoi_cert_paths() + assert set(paths) == {"ca_crt", "client_crt", "client_key"} + assert paths["ca_crt"] == "" + def test_chassis_module_entry_all_fields(self, policy_reader): """All three fields coexist in CHASSIS_MODULE|SWITCH-HOST.""" tbl = Table(None, bmcctld.CHASSIS_MODULE_TABLE) @@ -501,8 +916,9 @@ def test_no_critical_events(self, critical_event_checker): class TestGracefulShutdownHandler: def test_powering_off_state_set_before_gnoi(self, graceful_shutdown, chassis): - """STATE_DB device_power_state shows POWERING_OFF before gNOI shutdown is issued.""" graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = MagicMock(return_value=10) + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) captured = {} original = graceful_shutdown.controller._update_host_state def capture_first(power_state, device_status=None): @@ -510,38 +926,99 @@ def capture_first(power_state, device_status=None): captured['power_state'] = power_state return original(power_state, device_status) graceful_shutdown.controller._update_host_state = capture_first - graceful_shutdown._issue_gnoi_shutdown = MagicMock(return_value=False) - graceful_shutdown.execute() + requester = MagicMock() + requester.poll_status.return_value = _report( + "failed [bmc-req:{}]".format(TEST_REQUEST_ID), status=2) + graceful_shutdown.controller.power_off = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + graceful_shutdown.execute( + _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN), + MagicMock(return_value=requester)) assert captured.get('power_state') == bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN def test_shutdown_delay_zero_skips_gnoi(self, graceful_shutdown, chassis): graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = MagicMock(return_value=0) - graceful_shutdown.execute() - # set_admin_state(False) must be called on the Switch-Host module - assert chassis.switch_host.get_admin_state() is False + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + graceful_shutdown.controller.power_off = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + factory = MagicMock() + outcome = graceful_shutdown.execute( + _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN), factory) + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_TIMEOUT_ZERO, + True, + ) + factory.assert_not_called() + graceful_shutdown.controller.power_off.assert_called_once() def test_gnoi_fails_triggers_power_off(self, graceful_shutdown, chassis): graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = MagicMock(return_value=10) - graceful_shutdown._issue_gnoi_shutdown = MagicMock(return_value=False) - graceful_shutdown.execute() - assert chassis.switch_host.get_admin_state() is False + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + requester = MagicMock() + requester.send_halt.side_effect = bmcctld.GnoiRpcError("unreachable") + graceful_shutdown.controller.power_off = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + outcome = graceful_shutdown.execute( + _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN), + MagicMock(return_value=requester)) + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_RPC_FAILURE, + True, + ) + graceful_shutdown.controller.power_off.assert_called_once() def test_gnoi_success_and_host_goes_offline_still_calls_power_off(self, graceful_shutdown, chassis): - """Even after graceful OFFLINE, power_off is always issued to remove power.""" graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = MagicMock(return_value=10) - graceful_shutdown._issue_gnoi_shutdown = MagicMock(return_value=True) - chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) - result = graceful_shutdown.execute() - assert result is True + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + requester = MagicMock() + requester.poll_status.return_value = _report( + "complete [bmc-req:{}]".format(TEST_REQUEST_ID)) + states = [] + update_state = graceful_shutdown.controller._update_host_state + + def record_state(power_state, device_status=None): + states.append(power_state) + return update_state(power_state, device_status) + + graceful_shutdown.controller._update_host_state = MagicMock( + side_effect=record_state) + outcome = graceful_shutdown.execute( + _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN), + MagicMock(return_value=requester)) + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True) + assert states == [ + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN, + bmcctld.SWITCH_HOST_POWERING_OFF, + bmcctld.POWER_STATE_OFF, + ] assert chassis.switch_host.get_admin_state() is False + requester.close.assert_called_once() def test_gnoi_timeout_triggers_power_off(self, graceful_shutdown, chassis): - graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = MagicMock(return_value=10) - graceful_shutdown._issue_gnoi_shutdown = MagicMock(return_value=True) - # Simulate timeout: host never goes OFFLINE within shutdown_delay - with patch.object(graceful_shutdown.controller, '_verify_oper_status', return_value=False): - graceful_shutdown.execute() - assert chassis.switch_host.get_admin_state() is False + graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = MagicMock(return_value=1) + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + requester = MagicMock() + requester.poll_status.return_value = _report( + "active [bmc-req:{}]".format(TEST_REQUEST_ID), active=True) + graceful_shutdown.controller.power_off = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN) + with patch.object(operation.cancel, 'wait', side_effect=lambda _delay: False), \ + patch('bmcctld.time.monotonic', side_effect=[0, 0, 0, 0, 2]): + outcome = graceful_shutdown.execute( + operation, MagicMock(return_value=requester)) + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_DEADLINE, + True, + ) + graceful_shutdown.controller.power_off.assert_called_once() def test_get_switch_host_addr_default(self, graceful_shutdown): with patch('builtins.open', side_effect=FileNotFoundError): @@ -556,6 +1033,1081 @@ def test_get_switch_host_addr_from_bmc_json(self, graceful_shutdown, tmp_path): addr = graceful_shutdown._get_switch_host_addr() assert addr == "10.0.0.1" + def test_get_switch_host_gnoi_port_default(self, graceful_shutdown): + with patch('builtins.open', side_effect=FileNotFoundError): + port = graceful_shutdown._get_switch_host_gnoi_port() + assert port == 8080 + + def test_get_switch_host_gnoi_port_from_bmc_json(self, graceful_shutdown, tmp_path): + import json + bmc_json = tmp_path / "bmc.json" + bmc_json.write_text(json.dumps({"switch_host_gnmi_port": 8443})) + with patch.object(bmcctld, 'BMC_JSON_PATHS', [str(bmc_json)]): + port = graceful_shutdown._get_switch_host_gnoi_port() + assert port == 8443 + + @pytest.mark.parametrize("first_data,expected_addr,expected_port", [ + ({"bmc_if_addr": "10.0.0.1"}, "10.0.0.1", 8443), + ({"switch_host_gnmi_port": 9443}, "10.0.0.2", 9443), + ({"bmc_if_addr": "", "switch_host_gnmi_port": None}, "10.0.0.2", 8443), + ({"bmc_if_addr": "", "switch_host_gnmi_port": 0}, "10.0.0.2", 0), + ]) + def test_bmc_link_settings_keep_independent_precedence( + self, graceful_shutdown, tmp_path, first_data, expected_addr, + expected_port): + import json + first = tmp_path / "first.json" + second = tmp_path / "second.json" + broken = tmp_path / "broken.json" + first.write_text(json.dumps(first_data)) + second.write_text(json.dumps({ + "bmc_if_addr": "10.0.0.2", "switch_host_gnmi_port": 8443, + })) + broken.write_text("invalid json") + with patch.object(bmcctld, "BMC_JSON_PATHS", [ + str(tmp_path / "missing.json"), str(broken), str(first), str(second)]): + assert graceful_shutdown._get_switch_host_addr() == expected_addr + assert graceful_shutdown._get_switch_host_gnoi_port() == expected_port + + @pytest.mark.parametrize("method", ["execute", "execute_restart"]) + @pytest.mark.parametrize("graceful,reason", [ + (True, None), (False, bmcctld.OP_REASON_DEADLINE), + (False, bmcctld.OP_REASON_PREEMPTED), + ]) + @pytest.mark.parametrize("power_result", [ + bmcctld.PowerCallResult.CANCELLED, + bmcctld.PowerCallResult.NOT_CONFIRMED, + ]) + def test_shutdown_and_restart_share_off_failure_outcomes( + self, graceful_shutdown, method, graceful, reason, power_result): + action = (bmcctld.ACTION_GRACEFUL_RESTART if method == "execute_restart" + else bmcctld.ACTION_GRACEFUL_SHUTDOWN) + operation = _make_operation(action) + graceful_shutdown._run_shutdown_leg = MagicMock(return_value=(graceful, reason)) + graceful_shutdown.controller.power_off = MagicMock(return_value=power_result) + graceful_shutdown.controller.power_on = MagicMock() + graceful_shutdown._event_log = MagicMock() + + outcome = getattr(graceful_shutdown, method)(operation, MagicMock()) + + if reason == bmcctld.OP_REASON_PREEMPTED: + expected_result = bmcctld.OP_RESULT_PREEMPTED + graceful_shutdown.controller.power_off.assert_not_called() + else: + expected_result = (bmcctld.OP_RESULT_PREEMPTED + if power_result == bmcctld.PowerCallResult.CANCELLED + else bmcctld.OP_RESULT_POWER_OFF_FAILED) + graceful_shutdown.controller.power_off.assert_called_once_with( + operation, operation.cancel) + assert outcome == (expected_result, reason or "-", False) + graceful_shutdown._run_shutdown_leg.assert_called_once() + graceful_shutdown.controller.power_on.assert_not_called() + if expected_result == bmcctld.OP_RESULT_POWER_OFF_FAILED: + graceful_shutdown._event_log.log_error.assert_called_once_with( + "POWER_OFF_FAILED request_id={}".format(operation.request_id)) + else: + graceful_shutdown._event_log.log_error.assert_not_called() + + +# -------------------------------------------------------------------------- +# Tests: graceful qualification, report classification, and gNOI transport +# -------------------------------------------------------------------------- + +OUR_REQUEST_ID = "3f2b1c8a-1234-4abc-8def-0123456789ab" +OTHER_REQUEST_ID = "7a25d48e-9876-4fed-8cba-fedcba987654" + + +def _report(reason, active=False, method=3, status=1, message=""): + return SimpleNamespace( + reason=reason, + active=active, + method=method, + status=SimpleNamespace(status=status, message=message), + ) + + +class TestReportClassifier: + + @pytest.mark.parametrize( + "resp, expected", + [ + (_report("done [bmc-req:{}]".format(OUR_REQUEST_ID)), + ("graceful", None)), + (_report("active [bmc-req:{}]".format(OUR_REQUEST_ID), active=True), + ("keep_waiting", None)), + (_report("failed [bmc-req:{}]".format(OUR_REQUEST_ID), status=2), + ("forced", bmcctld.OP_REASON_CHECK_FAILED)), + (_report("failed [bmc-req:{}]".format(OUR_REQUEST_ID), status=2, + message="backend failure"), + ("forced", bmcctld.OP_REASON_BACKEND_ANSWERED)), + (_report("done [bmc-req:{}]".format(OUR_REQUEST_ID), + message="backend answer"), + ("forced", bmcctld.OP_REASON_BACKEND_ANSWERED)), + (_report("done [bmc-req:{}]".format(OTHER_REQUEST_ID)), + ("keep_waiting", None)), + (_report("done [bmc-req:{}0]".format(OUR_REQUEST_ID)), + ("keep_waiting", None)), + (_report("done [bmc-req:{}]".format(OUR_REQUEST_ID[:-1])), + ("keep_waiting", None)), + (_report("done without a tag"), ("keep_waiting", None)), + (_report("done [bmc-req:{0}] [bmc-req:{0}]".format(OUR_REQUEST_ID)), + ("keep_waiting", None)), + (_report("done [bmc-req:{}]".format(OUR_REQUEST_ID), method=1), + ("forced", bmcctld.OP_REASON_CHECK_FAILED)), + ], + ids=[ + "graceful", "active", "host-failure", "backend-failure", + "backend-success", "foreign", "longer-id", "shorter-id", + "untagged", "duplicate-tag", "wrong-method", + ], + ) + def test_n_report_attribution(self, resp, expected): + assert bmcctld.classify_report(resp, OUR_REQUEST_ID) == expected + + +class TestGracefulQualification: + + def _configure_qualified(self, monkeypatch, tmp_path, graceful_shutdown): + cert_dir = tmp_path / "certs" + cert_dir.mkdir() + cert_paths = {field: str(cert_dir / os.path.basename(default)) + for field, default in bmcctld.DEFAULT_GNOI_CERT_PATHS.items()} + for path in cert_paths.values(): + with open(path, "wb") as stream: + stream.write(b"certificate-data") + + monkeypatch.setattr(bmcctld.device_info, "is_switch_bmc", lambda: True) + return cert_paths + + def test_g_qualification_accepts_only_complete_positive_gate( + self, monkeypatch, tmp_path, graceful_shutdown): + cert_paths = self._configure_qualified(monkeypatch, tmp_path, graceful_shutdown) + with patch("builtins.open") as direct_open: + assert graceful_shutdown._is_graceful_qualified(cert_paths) is True + direct_open.assert_not_called() + + @pytest.mark.parametrize( + "failure", + ["wrong-role", "missing-cert", "empty-cert", "cert-race"], + ) + def test_g_qualification_rejects_incomplete_gate( + self, failure, monkeypatch, tmp_path, graceful_shutdown): + cert_paths = self._configure_qualified( + monkeypatch, tmp_path, graceful_shutdown) + + if failure == "wrong-role": + monkeypatch.setattr(bmcctld.device_info, "is_switch_bmc", lambda: False) + elif failure == "missing-cert": + os.unlink(cert_paths["client_crt"]) + elif failure == "empty-cert": + with open(cert_paths["client_crt"], "wb"): + pass + else: + monkeypatch.setattr(bmcctld.os.path, "isfile", lambda _path: True) + monkeypatch.setattr(bmcctld.os.path, "getsize", + MagicMock(side_effect=FileNotFoundError)) + + assert graceful_shutdown._is_graceful_qualified(cert_paths) is False + + def test_g_qualification_identity_exception_is_logged( + self, monkeypatch, tmp_path, graceful_shutdown): + cert_paths = self._configure_qualified( + monkeypatch, tmp_path, graceful_shutdown) + failure = RuntimeError("unexpected identity failure") + monkeypatch.setattr( + bmcctld.device_info, "is_switch_bmc", + MagicMock(side_effect=failure)) + graceful_shutdown.log_error = MagicMock() + + assert graceful_shutdown._is_graceful_qualified(cert_paths) is False + graceful_shutdown.log_error.assert_called_once() + assert str(failure) in graceful_shutdown.log_error.call_args.args[0] + + def test_g_qualification_is_reevaluated_between_operations( + self, monkeypatch, tmp_path, graceful_shutdown): + cert_paths = self._configure_qualified( + monkeypatch, tmp_path, graceful_shutdown) + os.unlink(cert_paths["client_crt"]) + assert graceful_shutdown._is_graceful_qualified(cert_paths) is False + with open(cert_paths["client_crt"], "wb") as stream: + stream.write(b"certificate-data") + assert graceful_shutdown._is_graceful_qualified(cert_paths) is True + + +class FakeGrpcError(Exception): + pass + + +class TestGnoiRequester: + + def _make_requester(self, monkeypatch, tmp_path): + cert_dir = tmp_path / "certs" + cert_dir.mkdir() + (cert_dir / "ca.crt").write_bytes(b"ca") + (cert_dir / "client.key").write_bytes(b"key") + (cert_dir / "client.crt").write_bytes(b"cert") + + grpc_api = SimpleNamespace( + RpcError=FakeGrpcError, + ssl_channel_credentials=MagicMock(return_value="credentials"), + ) + proto_api = SimpleNamespace( + HALT=3, + RebootStatus=SimpleNamespace(STATUS_SUCCESS=1), + RebootRequest=MagicMock(side_effect=lambda **kwargs: SimpleNamespace(**kwargs)), + RebootStatusRequest=MagicMock(return_value="status-request"), + ) + client = MagicMock() + client_factory = MagicMock(return_value=client) + + monkeypatch.setattr(bmcctld, "grpc", grpc_api) + monkeypatch.setattr(bmcctld, "system_pb2", proto_api) + monkeypatch.setattr(bmcctld, "GnoiClient", client_factory) + + requester = bmcctld.GnoiRequester( + "169.254.100.2", 8080, + {"ca_crt": str(cert_dir / "ca.crt"), + "client_crt": str(cert_dir / "client.crt"), + "client_key": str(cert_dir / "client.key")}, + bmcctld.GNOI_SERVER_NAME) + return requester, grpc_api, proto_api, client_factory, client, cert_dir + + def test_close_normalizes_any_channel_failure(self): + requester = bmcctld.GnoiRequester( + "169.254.100.2", 8080, dict(bmcctld.DEFAULT_GNOI_CERT_PATHS), + bmcctld.GNOI_SERVER_NAME) + client = MagicMock() + client.__exit__.side_effect = RuntimeError("channel teardown failed") + requester._client = client + with pytest.raises(bmcctld.GnoiRpcError, match="channel teardown failed"): + requester.close() + assert requester._client is None + requester.close() + client.__exit__.assert_called_once() + + def test_f_requester_uses_exact_mtls_and_rpc_shapes(self, monkeypatch, tmp_path): + requester, grpc_api, proto_api, client_factory, client, _ = \ + self._make_requester(monkeypatch, tmp_path) + + requester.open() + grpc_api.ssl_channel_credentials.assert_called_once_with( + root_certificates=b"ca", private_key=b"key", certificate_chain=b"cert") + client_factory.assert_called_once_with( + "169.254.100.2:8080", + options=(("grpc.ssl_target_name_override", "switch-host.bmc-link.sonic"),), + credentials="credentials", + ) + client.__enter__.assert_called_once_with() + + requester.send_halt("BMC pre-shutdown request [bmc-req:{}]".format(OUR_REQUEST_ID), 30) + proto_api.RebootRequest.assert_called_once_with( + method=3, + message="BMC pre-shutdown request [bmc-req:{}]".format(OUR_REQUEST_ID), + ) + reboot_request = client.system.Reboot.call_args.args[0] + assert reboot_request.method == 3 + assert reboot_request.message.endswith("[bmc-req:{}]".format(OUR_REQUEST_ID)) + client.system.Reboot.assert_called_once_with(reboot_request, timeout=30) + + requester.poll_status(10) + proto_api.RebootStatusRequest.assert_called_once_with() + client.system.RebootStatus.assert_called_once_with("status-request", timeout=10) + + requester.close() + client.__exit__.assert_called_once_with(None, None, None) + + @pytest.mark.parametrize( + "failure_point", + ["cert-read", "credentials", "client-open", "send", "poll", "close"], + ) + def test_f_transport_failures_are_normalized_once( + self, failure_point, monkeypatch, tmp_path): + requester, grpc_api, _, client_factory, client, cert_dir = \ + self._make_requester(monkeypatch, tmp_path) + + if failure_point == "cert-read": + (cert_dir / "ca.crt").unlink() + elif failure_point == "credentials": + grpc_api.ssl_channel_credentials.side_effect = ValueError("bad PEM") + elif failure_point == "client-open": + client.__enter__.side_effect = FakeGrpcError("channel failed") + else: + requester.open() + if failure_point == "send": + client.system.Reboot.side_effect = FakeGrpcError("send failed") + elif failure_point == "poll": + client.system.RebootStatus.side_effect = FakeGrpcError("poll failed") + else: + client.__exit__.side_effect = FakeGrpcError("close failed") + + with pytest.raises(bmcctld.GnoiRpcError): + if failure_point in ("cert-read", "credentials", "client-open"): + requester.open() + elif failure_point == "send": + requester.send_halt("message", 30) + elif failure_point == "poll": + requester.poll_status(10) + else: + requester.close() + + if failure_point == "credentials": + assert grpc_api.ssl_channel_credentials.call_count == 1 + elif failure_point == "client-open": + assert client.__enter__.call_count == 1 + elif failure_point == "send": + assert client.system.Reboot.call_count == 1 + elif failure_point == "poll": + assert client.system.RebootStatus.call_count == 1 + elif failure_point == "close": + assert client.__exit__.call_count == 1 + else: + client_factory.assert_not_called() + + +class TestConfiguredCertificatePaths: + + def _setup(self, monkeypatch, tmp_path, graceful_shutdown, chassis): + _, grpc_api, _, _, client, _ = TestGnoiRequester()._make_requester( + monkeypatch, tmp_path) + paths = { + "ca_crt": str(tmp_path / "trust" / "root.pem"), + "client_crt": str(tmp_path / "identity" / "certificate.pem"), + "client_key": str(tmp_path / "identity" / "private.key"), + } + for field, path in paths.items(): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as stream: + stream.write(field.encode()) + table = graceful_shutdown.policy_reader._thread_database.table( + "CONFIG_DB", "BMC_GNOI") + _set_table_entry(table, "certs", paths) + timeout_table = graceful_shutdown.policy_reader._thread_database.table( + "CONFIG_DB", bmcctld.CHASSIS_MODULE_TABLE) + _set_table_entry(timeout_table, bmcctld.SWITCH_HOST_MODULE_KEY, + {bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT: "10"}) + monkeypatch.setattr(bmcctld.device_info, "is_switch_bmc", lambda: True) + graceful_shutdown._get_switch_host_addr = MagicMock(return_value="169.254.100.2") + graceful_shutdown._get_switch_host_gnoi_port = MagicMock(return_value=8080) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + graceful_shutdown.controller.power_off = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + graceful_shutdown.controller.power_on = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + client.system.RebootStatus.return_value = _report( + "done [bmc-req:{}]".format(TEST_REQUEST_ID)) + return paths, table, grpc_api, client + + @pytest.mark.parametrize("action", [ + bmcctld.ACTION_GRACEFUL_SHUTDOWN, bmcctld.ACTION_GRACEFUL_RESTART, + ]) + def test_configured_files_reach_tls_through_operation( + self, action, monkeypatch, tmp_path, graceful_shutdown, chassis): + paths, table, grpc_api, client = self._setup( + monkeypatch, tmp_path, graceful_shutdown, chassis) + operation = _make_operation(action) + operation.cancel.wait = MagicMock(return_value=False) + execute = (graceful_shutdown.execute_restart + if action == bmcctld.ACTION_GRACEFUL_RESTART + else graceful_shutdown.execute) + with patch.object(table, "get", wraps=table.get) as read_row, \ + patch("builtins.open", wraps=builtins.open) as read_file: + outcome = execute(operation, bmcctld.GnoiRequester) + + assert outcome == (bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True) + read_row.assert_called_once_with("certs") + assert read_file.call_args_list == [ + call(paths["ca_crt"], "rb"), call(paths["client_key"], "rb"), + call(paths["client_crt"], "rb"), + ] + grpc_api.ssl_channel_credentials.assert_called_once_with( + root_certificates=b"ca_crt", private_key=b"client_key", + certificate_chain=b"client_crt") + client.system.Reboot.assert_called_once() + client.__exit__.assert_called_once_with(None, None, None) + graceful_shutdown.controller.power_off.assert_called_once_with( + operation, operation.cancel) + if action == bmcctld.ACTION_GRACEFUL_RESTART: + graceful_shutdown.controller.power_on.assert_called_once_with( + operation, operation.cancel) + else: + graceful_shutdown.controller.power_on.assert_not_called() + + def test_snapshot_is_shared_until_next_operation( + self, monkeypatch, tmp_path, graceful_shutdown, chassis): + paths, table, grpc_api, _ = self._setup( + monkeypatch, tmp_path, graceful_shutdown, chassis) + replacement = tmp_path / "replacement.pem" + replacement.write_bytes(b"new-ca") + qualify = graceful_shutdown._is_graceful_qualified + + def change_config_after_resolution(selected): + table.set("certs", FieldValuePairs([ + ("ca_crt", str(replacement)), ("extra", "ignored")])) + return qualify(selected) + + monkeypatch.setattr(graceful_shutdown, "_is_graceful_qualified", + change_config_after_resolution) + with patch.object(table, "get", wraps=table.get) as read_row: + for _ in range(2): + outcome = graceful_shutdown.execute( + _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN), + bmcctld.GnoiRequester) + assert outcome == (bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True) + assert read_row.call_args_list == [call("certs"), call("certs")] + assert grpc_api.ssl_channel_credentials.call_args_list == [ + call(root_certificates=b"ca_crt", private_key=b"client_key", + certificate_chain=b"client_crt"), + call(root_certificates=b"new-ca", private_key=b"client_key", + certificate_chain=b"client_crt"), + ] + + @pytest.mark.parametrize("field", ["ca_crt", "client_crt", "client_key"]) + @pytest.mark.parametrize("failure", ["empty", "relative", "missing", "empty-file"]) + def test_invalid_selected_path_does_not_fall_back( + self, field, failure, monkeypatch, tmp_path, graceful_shutdown, chassis): + paths, table, _, _ = self._setup( + monkeypatch, tmp_path, graceful_shutdown, chassis) + monkeypatch.setattr(bmcctld, "DEFAULT_GNOI_CERT_PATHS", dict(paths)) + empty_file = tmp_path / "empty.pem" + empty_file.touch() + invalid = {"empty": "", "relative": "relative.pem", + "missing": str(tmp_path / "missing.pem"), + "empty-file": str(empty_file)}[failure] + _set_table_entry(table, "certs", {field: invalid}) + factory = MagicMock() + operation = _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN) + assert graceful_shutdown.execute(operation, factory) == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, bmcctld.OP_REASON_NOT_QUALIFIED, True) + factory.assert_not_called() + graceful_shutdown.controller.power_off.assert_called_once_with( + operation, operation.cancel) + assert field in graceful_shutdown.log_warning.call_args.args[0] + + @pytest.mark.parametrize("failure", [FileNotFoundError, PermissionError]) + def test_selected_file_read_failure_uses_rpc_failure( + self, failure, monkeypatch, tmp_path, graceful_shutdown, chassis): + paths, _, grpc_api, client = self._setup( + monkeypatch, tmp_path, graceful_shutdown, chassis) + with patch("builtins.open", side_effect=failure("file changed")) as read_file: + outcome = graceful_shutdown.execute( + _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN), + bmcctld.GnoiRequester) + assert outcome == (bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_RPC_FAILURE, True) + read_file.assert_called_once_with(paths["ca_crt"], "rb") + grpc_api.ssl_channel_credentials.assert_not_called() + client.system.Reboot.assert_not_called() + + @pytest.mark.parametrize("action,confirmed_result", [ + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, bmcctld.OP_RESULT_SUCCESS_FORCED), + (bmcctld.ACTION_GRACEFUL_RESTART, bmcctld.OP_RESULT_POWER_ON_FAILED), + ]) + @pytest.mark.parametrize("power_result", [ + bmcctld.PowerCallResult.CONFIRMED, bmcctld.PowerCallResult.NOT_CONFIRMED, + ]) + @pytest.mark.parametrize("cancelled", [False, True]) + def test_db_read_error_uses_existing_worker_recovery( + self, action, confirmed_result, power_result, cancelled, + monkeypatch, tmp_path, graceful_shutdown, chassis): + _, table, _, _ = self._setup( + monkeypatch, tmp_path, graceful_shutdown, chassis) + table.get = MagicMock(side_effect=RuntimeError("DB unavailable")) + controller = graceful_shutdown.controller + controller.power_off.return_value = power_result + controller._update_host_state = MagicMock() + factory = MagicMock() + daemon = SimpleNamespace(graceful_shutdown=graceful_shutdown, + controller=controller, gnoi_requester_factory=factory) + runner = bmcctld.OperationRunner(daemon, queue.PriorityQueue(), itertools.count()) + runner.log_error = MagicMock() + operation = _make_operation(action) + if cancelled: + operation.cancel.set() + runner._run_worker(operation) + table.get.assert_called_once_with("certs") + factory.assert_not_called() + controller._update_host_state.assert_not_called() + controller.power_on.assert_not_called() + if cancelled: + controller.power_off.assert_not_called() + assert operation.outcome[0] == bmcctld.OP_RESULT_PREEMPTED + else: + controller.power_off.assert_called_once_with(operation, operation.cancel) + expected = (confirmed_result if power_result == bmcctld.PowerCallResult.CONFIRMED + else bmcctld.OP_RESULT_POWER_OFF_FAILED) + assert operation.outcome == ( + expected, bmcctld.OP_REASON_UNCLASSIFIED, + expected == bmcctld.OP_RESULT_SUCCESS_FORCED) + + +class FakeClock: + def __init__(self, now=0.0): + self.now = now + + def __call__(self): + return self.now + + def advance(self, seconds): + self.now += seconds + + +class TestGracefulShutdownOperation: + + @pytest.mark.parametrize("action", [ + bmcctld.ACTION_GRACEFUL_SHUTDOWN, bmcctld.ACTION_GRACEFUL_RESTART, + ]) + @pytest.mark.parametrize("config", ["fresh", "missing-field", "stored-zero"]) + def test_default_and_stored_timeout_drive_graceful_leg( + self, action, config, graceful_shutdown, chassis): + controller = graceful_shutdown.controller + table = controller.chassis_module_config_table + if config == "stored-zero": + table.set(bmcctld.SWITCH_HOST_MODULE_KEY, [ + (bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT, "0")]) + controller.initialize_chassis_module(bmcctld.ADMIN_UP) + if config == "missing-field": + table.hdel(bmcctld.SWITCH_HOST_MODULE_KEY, + bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT) + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + controller.power_off = MagicMock(return_value=bmcctld.PowerCallResult.CONFIRMED) + controller.power_on = MagicMock(return_value=bmcctld.PowerCallResult.CONFIRMED) + operation = _make_operation(action) + requester = MagicMock() + requester.poll_status.return_value = _report( + "active [bmc-req:{}]".format(TEST_REQUEST_ID), active=True) + factory = MagicMock(return_value=requester) + clock = FakeClock() + operation.cancel.wait = MagicMock( + side_effect=lambda delay: clock.advance(delay) or False) + execute = (graceful_shutdown.execute_restart + if action == bmcctld.ACTION_GRACEFUL_RESTART + else graceful_shutdown.execute) + with patch("bmcctld.time.monotonic", side_effect=clock): + outcome = execute(operation, factory) + + if config == "stored-zero": + expected_reason = bmcctld.OP_REASON_TIMEOUT_ZERO + factory.assert_not_called() + expected_wait = 0 + else: + expected_reason = bmcctld.OP_REASON_DEADLINE + factory.assert_called_once() + requester.send_halt.assert_called_once_with( + "BMC pre-shutdown request [bmc-req:{}]".format(TEST_REQUEST_ID), + timeout_secs=30) + assert requester.poll_status.call_count == 120 + requester.close.assert_called_once() + expected_wait = 120 + assert outcome == (bmcctld.OP_RESULT_SUCCESS_FORCED, expected_reason, True) + controller.power_off.assert_called_once_with(operation, operation.cancel) + if action == bmcctld.ACTION_GRACEFUL_RESTART: + expected_wait += 10 + controller.power_on.assert_called_once_with(operation, operation.cancel) + else: + controller.power_on.assert_not_called() + assert clock.now == expected_wait + + def _setup(self, graceful_shutdown, chassis, timeout=10, + power_result=None): + if power_result is None: + power_result = bmcctld.PowerCallResult.CONFIRMED + graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = \ + MagicMock(return_value=timeout) + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + graceful_shutdown.controller.power_off = MagicMock( + return_value=power_result) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN) + requester = MagicMock() + factory = MagicMock(return_value=requester) + return operation, requester, factory + + def test_g_not_qualified_precedes_timeout_zero( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup( + graceful_shutdown, chassis, timeout=0) + graceful_shutdown._is_graceful_qualified.return_value = False + outcome = graceful_shutdown.execute(operation, factory) + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_NOT_QUALIFIED, + True, + ) + factory.assert_not_called() + requester.send_halt.assert_not_called() + + def test_h_operation_start_race_already_off( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup(graceful_shutdown, chassis) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + outcome = graceful_shutdown.execute(operation, factory) + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_ALREADY_OFF, + True, + ) + factory.assert_not_called() + graceful_shutdown.controller.power_off.assert_called_once_with( + operation, operation.cancel) + + @pytest.mark.parametrize( + "response, expected_reason", + [ + (_report("failed [bmc-req:{}]".format(TEST_REQUEST_ID), status=2), + bmcctld.OP_REASON_CHECK_FAILED), + (_report("busy [bmc-req:{}]".format(TEST_REQUEST_ID), status=2, + message="Previous reboot is ongoing"), + bmcctld.OP_REASON_BACKEND_ANSWERED), + (_report("backend [bmc-req:{}]".format(TEST_REQUEST_ID), + message="backend answered"), + bmcctld.OP_REASON_BACKEND_ANSWERED), + ], + ids=["host-check-failed", "host-busy", "backend-success-shape"], + ) + def test_f_tagged_terminal_reports_force_with_exact_reason( + self, response, expected_reason, graceful_shutdown, chassis): + operation, requester, factory = self._setup(graceful_shutdown, chassis) + requester.poll_status.return_value = response + outcome = graceful_shutdown.execute(operation, factory) + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, expected_reason, True) + requester.poll_status.assert_called_once() + + def test_f_poll_rpc_failure_forces_without_retry( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup(graceful_shutdown, chassis) + requester.poll_status.side_effect = bmcctld.GnoiRpcError("poll failed") + outcome = graceful_shutdown.execute(operation, factory) + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_RPC_FAILURE, + True, + ) + requester.poll_status.assert_called_once() + requester.close.assert_called_once() + + @pytest.mark.parametrize( + "response", + [ + _report("foreign [bmc-req:{}]".format(OTHER_REQUEST_ID)), + _report("untagged terminal"), + _report("active [bmc-req:{}]".format(TEST_REQUEST_ID), active=True), + _report("two [bmc-req:{0}] [bmc-req:{0}]".format(TEST_REQUEST_ID)), + ], + ids=["foreign", "untagged", "active", "duplicate-tag"], + ) + def test_n_unattributed_reports_wait_to_deadline( + self, response, graceful_shutdown, chassis): + operation, requester, factory = self._setup( + graceful_shutdown, chassis, timeout=1) + requester.poll_status.return_value = response + clock = FakeClock() + operation.cancel.wait = MagicMock( + side_effect=lambda delay: clock.advance(delay) or False) + with patch('bmcctld.time.monotonic', side_effect=clock): + outcome = graceful_shutdown.execute(operation, factory) + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_DEADLINE, + True, + ) + requester.poll_status.assert_called_once() + + def test_f_poll_consuming_budget_starts_no_sleep_or_second_poll( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup( + graceful_shutdown, chassis, timeout=1) + clock = FakeClock() + def poll_status(timeout_secs): + assert timeout_secs == 1 + clock.now = 1 + return _report( + "active [bmc-req:{}]".format(TEST_REQUEST_ID), active=True) + requester.poll_status.side_effect = poll_status + operation.cancel.wait = MagicMock() + with patch('bmcctld.time.monotonic', side_effect=clock): + outcome = graceful_shutdown.execute(operation, factory) + assert outcome[1] == bmcctld.OP_REASON_DEADLINE + requester.poll_status.assert_called_once() + operation.cancel.wait.assert_not_called() + + def test_f_budget_consumed_during_classification_starts_no_sleep( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup( + graceful_shutdown, chassis, timeout=1) + clock = FakeClock() + requester.poll_status.return_value = _report("foreign") + operation.cancel.wait = MagicMock() + def classify(_response, _request_id): + clock.now = 1 + return "keep_waiting", None + with patch('bmcctld.time.monotonic', side_effect=clock), \ + patch('bmcctld.classify_report', side_effect=classify): + outcome = graceful_shutdown.execute(operation, factory) + assert outcome[1] == bmcctld.OP_REASON_DEADLINE + operation.cancel.wait.assert_not_called() + + @pytest.mark.parametrize( + "report_time, expected_result, expected_reason", + [ + (0.9, bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-"), + (1.1, bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_DEADLINE), + ], + ids=["before-deadline", "after-deadline"], + ) + def test_f_deadline_precedes_late_terminal_report( + self, report_time, expected_result, expected_reason, + graceful_shutdown, chassis): + operation, requester, factory = self._setup( + graceful_shutdown, chassis, timeout=1) + clock = FakeClock() + def poll_status(timeout_secs): + assert timeout_secs == 1 + clock.now = report_time + return _report("done [bmc-req:{}]".format(TEST_REQUEST_ID)) + requester.poll_status.side_effect = poll_status + with patch('bmcctld.time.monotonic', side_effect=clock): + outcome = graceful_shutdown.execute(operation, factory) + assert outcome == (expected_result, expected_reason, True) + + def test_p_cancel_before_open_sends_nothing( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup(graceful_shutdown, chassis) + operation.cancel.set() + outcome = graceful_shutdown.execute(operation, factory) + assert outcome == ( + bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, + False, + ) + requester.open.assert_not_called() + requester.send_halt.assert_not_called() + requester.close.assert_called_once() + graceful_shutdown.controller.power_off.assert_not_called() + + def test_p_cancel_during_open_sends_nothing( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup(graceful_shutdown, chassis) + requester.open.side_effect = operation.cancel.set + outcome = graceful_shutdown.execute(operation, factory) + assert outcome[0] == bmcctld.OP_RESULT_PREEMPTED + requester.send_halt.assert_not_called() + requester.poll_status.assert_not_called() + requester.close.assert_called_once() + graceful_shutdown.controller.power_off.assert_not_called() + + def test_p_cancel_inside_poll_allows_only_that_rpc( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup(graceful_shutdown, chassis) + def poll_status(timeout_secs): + operation.cancel.set() + return _report( + "active [bmc-req:{}]".format(TEST_REQUEST_ID), active=True) + requester.poll_status.side_effect = poll_status + outcome = graceful_shutdown.execute(operation, factory) + assert outcome[0] == bmcctld.OP_RESULT_PREEMPTED + requester.poll_status.assert_called_once() + graceful_shutdown.controller.power_off.assert_not_called() + + @pytest.mark.parametrize( + "poll_result", + ["success", "failure", "error"], + ) + def test_cancelled_poll_result_is_not_consumed( + self, poll_result, graceful_shutdown, chassis): + operation, requester, factory = self._setup(graceful_shutdown, chassis) + + def poll_status(timeout_secs): + operation.cancel.set() + if poll_result == "error": + raise bmcctld.GnoiRpcError("poll failed") + return _report( + "done [bmc-req:{}]".format(TEST_REQUEST_ID), + status=1 if poll_result == "success" else 2, + ) + + requester.poll_status.side_effect = poll_status + + outcome = graceful_shutdown.execute(operation, factory) + + assert outcome == ( + bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, + False, + ) + requester.poll_status.assert_called_once() + graceful_shutdown.controller.power_off.assert_not_called() + + def test_p_cancelled_power_off_preserves_completed_graceful_leg( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup(graceful_shutdown, chassis) + requester.poll_status.return_value = _report( + "done [bmc-req:{}]".format(TEST_REQUEST_ID)) + + def cancel_power_off(_operation, _cancel): + operation.cancel.set() + return bmcctld.PowerCallResult.CANCELLED + + graceful_shutdown.controller.power_off.side_effect = cancel_power_off + + outcome = graceful_shutdown.execute(operation, factory) + + assert outcome == (bmcctld.OP_RESULT_PREEMPTED, "-", False) + graceful_shutdown.controller.power_off.assert_called_once_with( + operation, operation.cancel) + + def test_w_power_off_failure_keeps_leg_reason( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup( + graceful_shutdown, chassis, + power_result=bmcctld.PowerCallResult.NOT_CONFIRMED) + requester.poll_status.return_value = _report( + "failed [bmc-req:{}]".format(TEST_REQUEST_ID), status=2) + outcome = graceful_shutdown.execute(operation, factory) + assert outcome == ( + bmcctld.OP_RESULT_POWER_OFF_FAILED, + bmcctld.OP_REASON_CHECK_FAILED, + False, + ) + + def test_requester_is_closed_when_classifier_raises( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup(graceful_shutdown, chassis) + requester.poll_status.return_value = _report( + "done [bmc-req:{}]".format(TEST_REQUEST_ID)) + with patch('bmcctld.classify_report', side_effect=RuntimeError("bug")): + with pytest.raises(RuntimeError): + graceful_shutdown.execute(operation, factory) + requester.close.assert_called_once() + graceful_shutdown.controller.power_off.assert_not_called() + + +class TestGracefulRestartOperation: + + def _setup(self, graceful_shutdown, chassis, timeout=0, + power_off_result=None, power_on_result=None): + if power_off_result is None: + power_off_result = bmcctld.PowerCallResult.CONFIRMED + if power_on_result is None: + power_on_result = bmcctld.PowerCallResult.CONFIRMED + graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = \ + MagicMock(return_value=timeout) + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + graceful_shutdown.controller.power_off = MagicMock( + return_value=power_off_result) + graceful_shutdown.controller.power_on = MagicMock( + return_value=power_on_result) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_RESTART) + operation.cancel.wait = MagicMock(return_value=False) + requester = MagicMock() + factory = MagicMock(return_value=requester) + return operation, requester, factory + + def test_h_graceful_restart_uses_two_steps_and_preserves_admin_status( + self, graceful_shutdown, chassis): + assert bmcctld.RESTART_PAUSE_SECS == 10 + controller = graceful_shutdown.controller + graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = \ + MagicMock(return_value=10) + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + original_set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=original_set_admin_state) + controller._update_host_state = MagicMock( + wraps=controller._update_host_state) + controller.chassis_module_config_table.set( + bmcctld.SWITCH_HOST_MODULE_KEY, + FieldValuePairs([(bmcctld.FIELD_ADMIN_STATUS, + bmcctld.ADMIN_DOWN)])) + admin_before = dict(controller.chassis_module_config_table.get( + bmcctld.SWITCH_HOST_MODULE_KEY)[1]) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_RESTART) + operation.cancel.wait = MagicMock(return_value=False) + requester = MagicMock() + requester.poll_status.return_value = _report( + "done [bmc-req:{}]".format(TEST_REQUEST_ID)) + factory = MagicMock(return_value=requester) + + outcome = graceful_shutdown.execute_restart(operation, factory) + + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True) + assert operation.stage == bmcctld.STAGE_POWER_ON_CONFIRMED + assert operation.cancel.wait.call_args_list == [ + call(bmcctld.RESTART_PAUSE_SECS)] + assert [args[0] for args, _ in + controller._update_host_state.call_args_list] == [ + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN, + bmcctld.SWITCH_HOST_POWERING_OFF, + bmcctld.POWER_STATE_OFF, + bmcctld.SWITCH_HOST_POWERING_ON, + bmcctld.POWER_STATE_ON, + ] + assert chassis.switch_host.set_admin_state.call_args_list == [ + call(False), call(True)] + assert chassis.switch_host.power_cycle_called is False + assert dict(controller.chassis_module_config_table.get( + bmcctld.SWITCH_HOST_MODULE_KEY)[1]) == admin_before + requester.send_halt.assert_called_once() + + def test_h_forced_restart_keeps_leg_reason_and_uses_no_cycle( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup( + graceful_shutdown, chassis) + + outcome = graceful_shutdown.execute_restart(operation, factory) + + assert outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_TIMEOUT_ZERO, + True, + ) + graceful_shutdown.controller.power_off.assert_called_once_with( + operation, operation.cancel) + operation.cancel.wait.assert_called_once_with( + bmcctld.RESTART_PAUSE_SECS) + graceful_shutdown.controller.power_on.assert_called_once_with( + operation, operation.cancel) + factory.assert_not_called() + requester.send_halt.assert_not_called() + assert chassis.switch_host.power_cycle_called is False + + def test_p_restart_handshake_cancellation_makes_no_power_call( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup( + graceful_shutdown, chassis, timeout=10) + operation.cancel.set() + + outcome = graceful_shutdown.execute_restart(operation, factory) + + assert outcome == ( + bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, + False, + ) + requester.open.assert_not_called() + requester.send_halt.assert_not_called() + graceful_shutdown.controller.power_off.assert_not_called() + graceful_shutdown.controller.power_on.assert_not_called() + + def test_p_restart_power_off_cancellation_stops_before_pause_and_raise( + self, graceful_shutdown, chassis): + operation, _, factory = self._setup( + graceful_shutdown, chassis, + power_off_result=bmcctld.PowerCallResult.CANCELLED) + + outcome = graceful_shutdown.execute_restart(operation, factory) + + assert outcome == ( + bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_TIMEOUT_ZERO, + False, + ) + operation.cancel.wait.assert_not_called() + graceful_shutdown.controller.power_on.assert_not_called() + + def test_p_restart_cancelled_during_pause_stays_off( + self, graceful_shutdown, chassis): + operation, requester, factory = self._setup( + graceful_shutdown, chassis, timeout=10) + requester.poll_status.return_value = _report( + "done [bmc-req:{}]".format(TEST_REQUEST_ID)) + operation.cancel.wait.return_value = True + + outcome = graceful_shutdown.execute_restart(operation, factory) + + assert outcome == (bmcctld.OP_RESULT_PREEMPTED, "-", False) + assert operation.stage == bmcctld.STAGE_PAUSE + graceful_shutdown.controller.power_on.assert_not_called() + + def test_p_restart_power_on_cancellation_retains_shutdown_leg_reason( + self, graceful_shutdown, chassis): + operation, _, factory = self._setup( + graceful_shutdown, chassis, + power_on_result=bmcctld.PowerCallResult.CANCELLED) + + outcome = graceful_shutdown.execute_restart(operation, factory) + + assert outcome == ( + bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_TIMEOUT_ZERO, + False, + ) + operation.cancel.wait.assert_called_once_with( + bmcctld.RESTART_PAUSE_SECS) + graceful_shutdown.controller.power_on.assert_called_once_with( + operation, operation.cancel) + + def test_w_restart_power_off_failure_stops_before_pause_and_raise( + self, graceful_shutdown, chassis): + operation, _, factory = self._setup( + graceful_shutdown, chassis, + power_off_result=bmcctld.PowerCallResult.NOT_CONFIRMED) + + outcome = graceful_shutdown.execute_restart(operation, factory) + + assert outcome == ( + bmcctld.OP_RESULT_POWER_OFF_FAILED, + bmcctld.OP_REASON_TIMEOUT_ZERO, + False, + ) + operation.cancel.wait.assert_not_called() + graceful_shutdown.controller.power_on.assert_not_called() + + def test_w_restart_power_on_failure_keeps_shutdown_leg_reason( + self, graceful_shutdown, chassis): + operation, _, factory = self._setup( + graceful_shutdown, chassis, + power_on_result=bmcctld.PowerCallResult.NOT_CONFIRMED) + + outcome = graceful_shutdown.execute_restart(operation, factory) + + assert outcome == ( + bmcctld.OP_RESULT_POWER_ON_FAILED, + bmcctld.OP_REASON_TIMEOUT_ZERO, + False, + ) + graceful_shutdown.controller.power_on.assert_called_once_with( + operation, operation.cancel) + + def test_p_restart_raise_is_refused_by_under_lock_critical_read( + self, graceful_shutdown, chassis): + controller = graceful_shutdown.controller + graceful_shutdown.policy_reader.get_graceful_shutdown_timeout = \ + MagicMock(return_value=0) + graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + original_set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=original_set_admin_state) + controller.critical_event_checker = MagicMock() + controller.critical_event_checker.has_any_critical_event.return_value = True + operation = _make_operation(bmcctld.ACTION_GRACEFUL_RESTART) + operation.cancel.wait = MagicMock(return_value=False) + + outcome = graceful_shutdown.execute_restart(operation, MagicMock()) + + assert outcome == ( + bmcctld.OP_RESULT_OFF_LEAK_BLOCKED, + bmcctld.OP_REASON_TIMEOUT_ZERO, + False, + ) + assert chassis.switch_host.set_admin_state.call_args_list == [ + call(False)] + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_OFFLINE + state = dict(controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_DEVICE_STATUS] == \ + bmcctld.SWITCH_HOST_OFFLINE + controller.critical_event_checker.has_any_critical_event.assert_called_once() + # -------------------------------------------------------------------------- # Tests: BmcEventHandler - Rack Manager commands @@ -568,15 +2120,19 @@ def _cmd_fvs(self, command, status=bmcctld.CMD_STATUS_PENDING): def test_power_off_command_enqueues_power_off(self, event_handler): event_handler._handle_rack_mgr_command("CMD_1", self._cmd_fvs(bmcctld.CMD_POWER_OFF)) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_POWER_OFF + assert item.priority == 2 + assert item.rack_cmd_key == "CMD_1" assert item.on_complete is not None def test_power_on_command_no_leak_enqueues_power_on(self, event_handler): event_handler.critical_event_checker.has_any_critical_event = MagicMock(return_value=False) event_handler._handle_rack_mgr_command("CMD_2", self._cmd_fvs(bmcctld.CMD_POWER_ON)) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_POWER_ON + assert item.priority == 5 + assert item.rack_cmd_key == "CMD_2" assert item.on_complete is not None def test_power_on_command_blocked_by_critical_leak(self, event_handler): @@ -586,17 +2142,87 @@ def test_power_on_command_blocked_by_critical_leak(self, event_handler): def test_power_cycle_command_enqueues_power_cycle(self, event_handler): event_handler._handle_rack_mgr_command("CMD_4", self._cmd_fvs(bmcctld.CMD_POWER_CYCLE)) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_POWER_CYCLE + assert item.priority == 4 + assert item.rack_cmd_key == "CMD_4" + + def test_q_power_cycle_command_blocked_by_critical_leak(self, event_handler): + event_handler.critical_event_checker.has_any_critical_event = \ + MagicMock(return_value=True) + event_handler._handle_rack_mgr_command( + "CMD_4B", self._cmd_fvs(bmcctld.CMD_POWER_CYCLE)) + assert event_handler.action_queue.empty() def test_already_processed_command_is_skipped(self, event_handler): + event_handler._set_cmd_status = MagicMock() event_handler._handle_rack_mgr_command( "CMD_5", self._cmd_fvs(bmcctld.CMD_POWER_ON, bmcctld.CMD_STATUS_DONE)) assert event_handler.action_queue.empty() + event_handler._set_cmd_status.assert_not_called() def test_unknown_command_is_logged(self, event_handler): + event_handler._set_cmd_status = MagicMock(wraps=event_handler._set_cmd_status) event_handler._handle_rack_mgr_command("CMD_6", self._cmd_fvs("INVALID_CMD")) assert event_handler.action_queue.empty() + assert event_handler._set_cmd_status.call_args_list == [ + call("CMD_6", bmcctld.CMD_STATUS_IN_PROGRESS), + call("CMD_6", bmcctld.CMD_STATUS_FAILED, "UNKNOWN_COMMAND"), + ] + + def test_command_callbacks_preserve_their_own_rows(self, event_handler): + commands = [ + (bmcctld.CMD_POWER_ON, bmcctld.ACTION_POWER_ON, 5), + (bmcctld.CMD_POWER_OFF, bmcctld.ACTION_POWER_OFF, 2), + (bmcctld.CMD_POWER_CYCLE, bmcctld.ACTION_POWER_CYCLE, 4), + (bmcctld.CMD_GRACEFUL_SHUT, bmcctld.ACTION_GRACEFUL_SHUTDOWN, 3), + (bmcctld.CMD_GRACEFUL_RESTART, bmcctld.ACTION_GRACEFUL_RESTART, 4), + ] + leak_check = MagicMock(return_value=False) + event_handler.critical_event_checker.has_any_critical_event = leak_check + table = event_handler._thread_database.table( + "STATE_DB", bmcctld.RACK_MANAGER_COMMAND_TABLE) + callbacks = [] + for index, (command, action, priority) in enumerate(commands): + key = "CMD_{}".format(index) + fields = self._cmd_fvs(command) + fields["requester"] = "test-{}".format(index) + fields[bmcctld.FIELD_REQUEST_ID] = "request-{}".format(index) + table.set(key, FieldValuePairs(list(fields.items()))) + event_handler._handle_rack_mgr_command(key, fields) + item = _dequeue_item(event_handler.action_queue) + assert (item.action, item.priority, item.rack_cmd_key) == (action, priority, key) + assert item.event_desc == "RACK_MGR_CMD:" + command + assert dict(table.get(key)[1])[bmcctld.FIELD_STATUS] == bmcctld.CMD_STATUS_IN_PROGRESS + callbacks.append((item.on_complete, key, fields, index)) + assert leak_check.call_count == 2 + + for callback, key, fields, index in reversed(callbacks): + success = index % 2 == 0 + detail = "BUSY" if index == 1 else "" + callback(success, detail) + row = dict(table.get(key)[1]) + assert row[bmcctld.FIELD_STATUS] == ( + bmcctld.CMD_STATUS_DONE if success else bmcctld.CMD_STATUS_FAILED) + assert row[bmcctld.FIELD_RESULT] == ( + "SUCCESS" if success else (detail or "ERROR")) + for field in (bmcctld.FIELD_COMMAND, bmcctld.FIELD_REQUEST_ID, "requester"): + assert row[field] == fields[field] + + def test_q_graceful_restart_command_is_admitted_without_early_leak_gate( + self, event_handler): + event_handler.critical_event_checker.has_any_critical_event = \ + MagicMock(return_value=True) + + event_handler._handle_rack_mgr_command( + "CMD_RESTART", self._cmd_fvs(bmcctld.CMD_GRACEFUL_RESTART)) + + item = _dequeue_item(event_handler.action_queue) + assert item.action == bmcctld.ACTION_GRACEFUL_RESTART + assert item.priority == 4 + assert item.rack_cmd_key == "CMD_RESTART" + leak_check = event_handler.critical_event_checker.has_any_critical_event + leak_check.assert_not_called() # -------------------------------------------------------------------------- @@ -613,18 +2239,21 @@ def test_admin_down_triggers_graceful_shutdown_when_online(self, event_handler, bmcctld.SWITCH_HOST_MODULE_KEY, {bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_DOWN}, ) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_GRACEFUL_SHUTDOWN + assert item.priority == 3 - def test_admin_down_no_action_when_already_offline(self, event_handler, controller): - # host is OFFLINE (e.g. startup replay) → admin_down should be a no-op + def test_admin_down_is_admitted_when_already_offline( + self, event_handler, controller): _set_table_entry(controller.host_state_table, bmcctld.HOST_STATE_KEY, {bmcctld.FIELD_DEVICE_STATUS: bmcctld.SWITCH_HOST_OFFLINE}) event_handler._handle_chassis_module( bmcctld.SWITCH_HOST_MODULE_KEY, {bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_DOWN}, ) - assert event_handler.action_queue.empty() + item = _dequeue_item(event_handler.action_queue) + assert item.action == bmcctld.ACTION_GRACEFUL_SHUTDOWN + assert item.priority == 3 def test_admin_up_powers_on_when_no_leak(self, event_handler, controller): # host is OFFLINE → admin_up should enqueue power_on @@ -635,8 +2264,9 @@ def test_admin_up_powers_on_when_no_leak(self, event_handler, controller): bmcctld.SWITCH_HOST_MODULE_KEY, {bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_UP}, ) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_POWER_ON + assert item.priority == 5 def test_admin_up_no_action_when_already_online(self, event_handler, controller): # host is already ONLINE → admin_up should be a no-op @@ -677,7 +2307,7 @@ def test_empty_admin_status_does_not_poison_dedup(self, event_handler, controlle bmcctld.SWITCH_HOST_MODULE_KEY, {bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_UP}, ) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_POWER_ON def test_seed_chassis_module_admin_status_ignores_config_replay(self, event_handler, controller): @@ -727,8 +2357,22 @@ def test_critical_system_leak_power_off(self, event_handler, chassis): bmcctld.SYSTEM_LEAK_STATUS_KEY, {bmcctld.FIELD_DEVICE_LEAK_STATUS: bmcctld.SYSTEM_LEAK_CRITICAL}, ) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_POWER_OFF + assert item.priority == 0 + + def test_critical_rack_alert_graceful_has_priority_one(self, event_handler): + event_handler.policy_reader.get_leak_control_policy = MagicMock( + return_value=self._make_policy( + rack_mgr_critical_alert_action=bmcctld.ACTION_GRACEFUL_SHUTDOWN) + ) + event_handler._handle_rack_mgr_alert( + "Rack_level_leak", + {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_CRITICAL}, + ) + item = _dequeue_item(event_handler.action_queue) + assert item.action == bmcctld.ACTION_GRACEFUL_SHUTDOWN + assert item.priority == 1 def test_critical_system_leak_graceful_shutdown(self, event_handler): event_handler.policy_reader.get_leak_control_policy = MagicMock( @@ -738,7 +2382,7 @@ def test_critical_system_leak_graceful_shutdown(self, event_handler): bmcctld.SYSTEM_LEAK_STATUS_KEY, {bmcctld.FIELD_DEVICE_LEAK_STATUS: bmcctld.SYSTEM_LEAK_CRITICAL}, ) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_GRACEFUL_SHUTDOWN def test_critical_system_leak_syslog_only(self, event_handler, chassis): @@ -822,8 +2466,9 @@ def test_critical_rack_alert_power_off_when_configured(self, event_handler, chas "Rack_level_leak", {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_CRITICAL}, ) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_POWER_OFF + assert item.priority == 0 def test_minor_rack_alert_syslog_only_by_default(self, event_handler): event_handler.policy_reader.get_leak_control_policy = MagicMock( @@ -868,8 +2513,9 @@ def test_rack_mgr_leak_policy_disabled_does_not_poison_dedup(self, event_handler "Inlet_liquid_pressure", {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_CRITICAL}, ) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_POWER_OFF + assert item.priority == 0 assert event_handler._last_rack_mgr_alert_severity["Inlet_liquid_pressure"] == \ bmcctld.ALERT_SEVERITY_CRITICAL @@ -881,7 +2527,7 @@ def test_rack_level_leak_uses_leak_field(self, event_handler): "Rack_level_leak", {bmcctld.FIELD_LEAK: bmcctld.ALERT_SEVERITY_CRITICAL}, ) - item = event_handler.action_queue.get_nowait() + item = _dequeue_item(event_handler.action_queue) assert item.action == bmcctld.ACTION_POWER_OFF def test_normal_severity_no_action(self, event_handler): @@ -900,95 +2546,92 @@ class TestBmcEventHandlerRackMgrData: def test_data_subscriber_registered(self, event_handler): """RACK_MANAGER_DATA is subscribed and routed to _handle_rack_mgr_data.""" - fd = event_handler.rack_mgr_data_sub.getFd() - assert fd in event_handler._subscribers - assert event_handler._subscribers[fd][1] == event_handler._handle_rack_mgr_data + _selector, subscribers = event_handler._create_event_subscriptions() + routed = [ + (subscriber.table_name, connector) + for subscriber, handler, connector in subscribers.values() + if handler == event_handler._handle_rack_mgr_data + ] + assert routed == [(bmcctld.RACK_MANAGER_DATA_TABLE, "STATE_DB")] def test_normal_telemetry_clears_active_alert(self, event_handler): """NORMAL telemetry deletes the matching RACK_MANAGER_ALERT row and resets dedup.""" - tbl = Table(None, bmcctld.RACK_MANAGER_ALERT_TABLE) + tbl = Table("STATE_DB", bmcctld.RACK_MANAGER_ALERT_TABLE) _set_table_entry(tbl, "Inlet_liquid_temperature", {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_CRITICAL}) event_handler._last_rack_mgr_alert_severity["Inlet_liquid_temperature"] = \ bmcctld.ALERT_SEVERITY_CRITICAL - with patch.object(bmcctld.swsscommon, 'Table', return_value=tbl): - event_handler._handle_rack_mgr_data( - "Inlet_liquid_temperature", - {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_NORMAL}, - ) + event_handler._handle_rack_mgr_data( + "Inlet_liquid_temperature", + {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_NORMAL}, + ) assert "Inlet_liquid_temperature" not in tbl.mock_dict assert "Inlet_liquid_temperature" not in event_handler._last_rack_mgr_alert_severity def test_normal_telemetry_no_active_alert_is_noop(self, event_handler): """NORMAL telemetry with no active alert does nothing (no error, no delete).""" - tbl = Table(None, bmcctld.RACK_MANAGER_ALERT_TABLE) - with patch.object(bmcctld.swsscommon, 'Table', return_value=tbl): - event_handler._handle_rack_mgr_data( - "Inlet_liquid_pressure", - {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_NORMAL}, - ) + tbl = Table("STATE_DB", bmcctld.RACK_MANAGER_ALERT_TABLE) + event_handler._handle_rack_mgr_data( + "Inlet_liquid_pressure", + {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_NORMAL}, + ) assert tbl.mock_dict == {} def test_non_normal_telemetry_leaves_alert(self, event_handler): """Non-NORMAL telemetry never clears an alert (alerts are raised via ALERT table).""" - tbl = Table(None, bmcctld.RACK_MANAGER_ALERT_TABLE) + tbl = Table("STATE_DB", bmcctld.RACK_MANAGER_ALERT_TABLE) _set_table_entry(tbl, "Inlet_liquid_flow_rate", {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_CRITICAL}) event_handler._last_rack_mgr_alert_severity["Inlet_liquid_flow_rate"] = \ bmcctld.ALERT_SEVERITY_CRITICAL - with patch.object(bmcctld.swsscommon, 'Table', return_value=tbl): - event_handler._handle_rack_mgr_data( - "Inlet_liquid_flow_rate", - {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_MINOR}, - ) + event_handler._handle_rack_mgr_data( + "Inlet_liquid_flow_rate", + {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_MINOR}, + ) assert "Inlet_liquid_flow_rate" in tbl.mock_dict assert event_handler._last_rack_mgr_alert_severity["Inlet_liquid_flow_rate"] == \ bmcctld.ALERT_SEVERITY_CRITICAL def test_rack_level_leak_telemetry_clears_via_leak_field(self, event_handler): """Rack_level_leak telemetry reports NORMAL via the 'leak' field and clears the alert.""" - tbl = Table(None, bmcctld.RACK_MANAGER_ALERT_TABLE) + tbl = Table("STATE_DB", bmcctld.RACK_MANAGER_ALERT_TABLE) _set_table_entry(tbl, "Rack_level_leak", {bmcctld.FIELD_LEAK: bmcctld.ALERT_SEVERITY_CRITICAL}) - with patch.object(bmcctld.swsscommon, 'Table', return_value=tbl): - event_handler._handle_rack_mgr_data( - "Rack_level_leak", - {bmcctld.FIELD_LEAK: bmcctld.ALERT_SEVERITY_NORMAL}, - ) + event_handler._handle_rack_mgr_data( + "Rack_level_leak", + {bmcctld.FIELD_LEAK: bmcctld.ALERT_SEVERITY_NORMAL}, + ) assert "Rack_level_leak" not in tbl.mock_dict def test_mixed_case_normal_clears(self, event_handler): """A mixed-case 'Normal' telemetry severity still clears the alert.""" - tbl = Table(None, bmcctld.RACK_MANAGER_ALERT_TABLE) + tbl = Table("STATE_DB", bmcctld.RACK_MANAGER_ALERT_TABLE) _set_table_entry(tbl, "Inlet_liquid_pressure", {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_CRITICAL}) - with patch.object(bmcctld.swsscommon, 'Table', return_value=tbl): - event_handler._handle_rack_mgr_data( - "Inlet_liquid_pressure", - {bmcctld.FIELD_SEVERITY: "Normal"}, - ) + event_handler._handle_rack_mgr_data( + "Inlet_liquid_pressure", + {bmcctld.FIELD_SEVERITY: "Normal"}, + ) assert "Inlet_liquid_pressure" not in tbl.mock_dict def test_unknown_telemetry_key_ignored(self, event_handler): """Telemetry under a key outside ALERT_KEYS is ignored (no table access, no error).""" - tbl = Table(None, bmcctld.RACK_MANAGER_ALERT_TABLE) + tbl = Table("STATE_DB", bmcctld.RACK_MANAGER_ALERT_TABLE) _set_table_entry(tbl, "Inlet_liquid_temperature", {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_CRITICAL}) - with patch.object(bmcctld.swsscommon, 'Table', return_value=tbl): - event_handler._handle_rack_mgr_data( - "Some_unexpected_key", - {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_NORMAL}, - ) + event_handler._handle_rack_mgr_data( + "Some_unexpected_key", + {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_NORMAL}, + ) # Unrelated active alert must be untouched. assert "Inlet_liquid_temperature" in tbl.mock_dict def test_missing_fields_telemetry_does_not_clear_alert(self, event_handler): """Malformed telemetry (no severity and no leak field) must not clear an alert.""" - tbl = Table(None, bmcctld.RACK_MANAGER_ALERT_TABLE) + tbl = Table("STATE_DB", bmcctld.RACK_MANAGER_ALERT_TABLE) _set_table_entry(tbl, "Inlet_liquid_temperature", {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_CRITICAL}) - with patch.object(bmcctld.swsscommon, 'Table', return_value=tbl): - event_handler._handle_rack_mgr_data("Inlet_liquid_temperature", {}) + event_handler._handle_rack_mgr_data("Inlet_liquid_temperature", {}) assert "Inlet_liquid_temperature" in tbl.mock_dict @@ -1005,75 +2648,115 @@ def _make_daemon(self, chassis): daemon.policy_reader.get_power_on_delay = MagicMock(return_value=0) return daemon + def test_gnoi_requester_factory_is_injectable(self, chassis): + daemon = self._make_daemon(chassis) + assert daemon.gnoi_requester_factory is bmcctld.GnoiRequester + replacement = MagicMock() + daemon.gnoi_requester_factory = replacement + assert daemon.gnoi_requester_factory is replacement + def test_execute_graceful_shutdown(self, chassis): daemon = self._make_daemon(chassis) - chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) - daemon.graceful_shutdown.execute = MagicMock(return_value=True) - item = bmcctld.ActionItem(bmcctld.ACTION_GRACEFUL_SHUTDOWN, "test") - daemon._execute_action_item(item) + daemon.graceful_shutdown.execute = MagicMock(return_value=( + bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True)) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN) + daemon.operation_runner._run_worker(operation) daemon.graceful_shutdown.execute.assert_called_once() + assert operation.outcome == ( + bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True) + + def test_execute_graceful_restart(self, chassis): + daemon = self._make_daemon(chassis) + daemon.graceful_shutdown.execute_restart = MagicMock(return_value=( + bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True)) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_RESTART) + + daemon.operation_runner._run_worker(operation) + + daemon.graceful_shutdown.execute_restart.assert_called_once_with( + operation, daemon.gnoi_requester_factory) + assert operation.outcome == ( + bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True) def test_execute_power_off(self, chassis): daemon = self._make_daemon(chassis) - chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) - daemon.controller.power_off = MagicMock(return_value=True) - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_OFF, "test") - daemon._execute_action_item(item) + daemon.controller.power_off = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + operation = _make_operation(bmcctld.ACTION_POWER_OFF) + daemon.operation_runner._run_worker(operation) daemon.controller.power_off.assert_called_once() + assert operation.outcome == (bmcctld.OP_RESULT_SUCCESS, "-", True) def test_execute_power_on(self, chassis): daemon = self._make_daemon(chassis) - daemon.controller.power_on = MagicMock(return_value=True) - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_ON, "test") - daemon._execute_action_item(item) + daemon.controller.power_on = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + operation = _make_operation(bmcctld.ACTION_POWER_ON) + daemon.operation_runner._run_worker(operation) daemon.controller.power_on.assert_called_once() + assert operation.outcome == (bmcctld.OP_RESULT_SUCCESS, "-", True) def test_execute_power_cycle(self, chassis): daemon = self._make_daemon(chassis) - daemon.controller.power_cycle = MagicMock(return_value=True) - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_CYCLE, "test") - daemon._execute_action_item(item) + daemon.controller.power_cycle = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + operation = _make_operation(bmcctld.ACTION_POWER_CYCLE) + daemon.operation_runner._run_worker(operation) daemon.controller.power_cycle.assert_called_once() + assert operation.outcome == (bmcctld.OP_RESULT_SUCCESS, "-", True) def test_on_complete_called_with_success(self, chassis): daemon = self._make_daemon(chassis) - daemon.controller.power_on = MagicMock(return_value=True) + daemon.controller.power_on = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) callback = MagicMock() - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_ON, "test", on_complete=callback) - daemon._execute_action_item(item) - callback.assert_called_once_with(True) + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "test", + bmcctld.action_priority(bmcctld.ACTION_POWER_ON), + on_complete=callback) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join() + daemon.operation_runner.process_next(timeout=0) + callback.assert_called_once_with(True, bmcctld.OP_RESULT_SUCCESS) def test_on_complete_called_with_failure(self, chassis): daemon = self._make_daemon(chassis) - daemon.controller.power_on = MagicMock(return_value=False) + daemon.controller.power_on = MagicMock( + return_value=bmcctld.PowerCallResult.NOT_CONFIRMED) callback = MagicMock() - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_ON, "test", on_complete=callback) - daemon._execute_action_item(item) - callback.assert_called_once_with(False) + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "test", + bmcctld.action_priority(bmcctld.ACTION_POWER_ON), + on_complete=callback) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join() + daemon.operation_runner.process_next(timeout=0) + callback.assert_called_once_with( + False, bmcctld.OP_RESULT_POWER_ON_FAILED) def test_action_loop_processes_queued_items(self, chassis): daemon = self._make_daemon(chassis) - # power_on mock must also flip oper_status so the subsequent power_off - # idempotency check sees ONLINE and proceeds instead of skipping. - def fake_power_on(): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + def power_off(_operation, _cancel): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + return bmcctld.PowerCallResult.CONFIRMED + def power_on(_operation, _cancel): chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) - return True - daemon.controller.power_on = MagicMock(side_effect=fake_power_on) - daemon.controller.power_off = MagicMock(return_value=True) - daemon.action_queue.put(bmcctld.ActionItem(bmcctld.ACTION_POWER_ON, "evt1")) - daemon.action_queue.put(bmcctld.ActionItem(bmcctld.ACTION_POWER_OFF, "evt2")) - - # Stop the loop after both items are executed - original = daemon._execute_action_item - calls = [0] - def counting_execute(item): - original(item) - calls[0] += 1 - if calls[0] >= 2: - daemon.stop_event.set() - daemon._execute_action_item = counting_execute - - daemon._run_action_loop() + return bmcctld.PowerCallResult.CONFIRMED + daemon.controller.power_off = MagicMock(side_effect=power_off) + daemon.controller.power_on = MagicMock(side_effect=power_on) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "evt1", 5)) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "evt2", 2)) + + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join() + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join() + daemon.operation_runner.process_next(timeout=0) daemon.controller.power_on.assert_called_once() daemon.controller.power_off.assert_called_once() @@ -1083,41 +2766,62 @@ def test_execute_power_off_skipped_when_already_offline(self, chassis): """power_off is not issued when host is already OFFLINE; on_complete(True) fired.""" # chassis.switch_host starts OFFLINE by default daemon = self._make_daemon(chassis) - daemon.controller.power_off = MagicMock(return_value=True) + daemon.controller.init_host_state() + daemon.controller.power_off = MagicMock() callback = MagicMock() - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_OFF, "dup-event", on_complete=callback) - daemon._execute_action_item(item) + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "dup-event", 2, + on_complete=callback) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) daemon.controller.power_off.assert_not_called() - callback.assert_called_once_with(True) + callback.assert_called_once_with(True, "guard_skip") + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS def test_execute_graceful_shutdown_skipped_when_already_offline(self, chassis): """graceful_shutdown is not issued when host is already OFFLINE.""" daemon = self._make_daemon(chassis) - daemon.graceful_shutdown.execute = MagicMock(return_value=True) + daemon.controller.init_host_state() + daemon.graceful_shutdown.execute = MagicMock() callback = MagicMock() - item = bmcctld.ActionItem(bmcctld.ACTION_GRACEFUL_SHUTDOWN, "dup-event", on_complete=callback) - daemon._execute_action_item(item) + item = bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "dup-event", 3, + on_complete=callback) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) daemon.graceful_shutdown.execute.assert_not_called() - callback.assert_called_once_with(True) + callback.assert_called_once_with(True, "guard_skip") + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS_FORCED + assert state[bmcctld.FIELD_OP_REASON] == bmcctld.OP_REASON_ALREADY_OFF def test_execute_power_on_skipped_when_already_online(self, chassis): """power_on is not issued when host is already ONLINE; on_complete(True) fired.""" chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) daemon = self._make_daemon(chassis) - daemon.controller.power_on = MagicMock(return_value=True) + daemon.controller.power_on = MagicMock() callback = MagicMock() - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_ON, "dup-event", on_complete=callback) - daemon._execute_action_item(item) + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "dup-event", 5, + on_complete=callback) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) daemon.controller.power_on.assert_not_called() - callback.assert_called_once_with(True) + callback.assert_called_once_with(True, "guard_skip") def test_execute_power_cycle_not_skipped_when_online(self, chassis): """power_cycle always executes regardless of current oper_status.""" chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) daemon = self._make_daemon(chassis) - daemon.controller.power_cycle = MagicMock(return_value=True) - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_CYCLE, "test") - daemon._execute_action_item(item) + daemon.controller.power_cycle = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + item = bmcctld.ActionItem(bmcctld.ACTION_POWER_CYCLE, "test", 4) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join() daemon.controller.power_cycle.assert_called_once() def test_execute_power_off_skipped_when_powering_off_in_progress(self, chassis): @@ -1126,36 +2830,48 @@ def test_execute_power_off_skipped_when_powering_off_in_progress(self, chassis): daemon = self._make_daemon(chassis) # Simulate a power_off already in progress by writing transitional power state to DB daemon.controller._update_host_state(bmcctld.SWITCH_HOST_POWERING_OFF) - daemon.controller.power_off = MagicMock(return_value=True) + daemon.controller.write_operation_start(TEST_REQUEST_ID, "active") + daemon.controller.power_off = MagicMock() callback = MagicMock() - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_OFF, "dup-leak-event", on_complete=callback) - daemon._execute_action_item(item) + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "dup-leak-event", 2, + on_complete=callback) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) daemon.controller.power_off.assert_not_called() - callback.assert_called_once_with(True) + callback.assert_called_once_with(True, "guard_skip") def test_execute_graceful_shutdown_skipped_when_powering_off_in_progress(self, chassis): """graceful_shutdown is skipped when STATE_DB device_power_state shows POWERING_OFF.""" chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) daemon = self._make_daemon(chassis) daemon.controller._update_host_state(bmcctld.SWITCH_HOST_POWERING_OFF) - daemon.graceful_shutdown.execute = MagicMock(return_value=True) + daemon.controller.write_operation_start(TEST_REQUEST_ID, "active") + daemon.graceful_shutdown.execute = MagicMock() callback = MagicMock() - item = bmcctld.ActionItem(bmcctld.ACTION_GRACEFUL_SHUTDOWN, "dup-cmd", on_complete=callback) - daemon._execute_action_item(item) + item = bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "dup-cmd", 3, + on_complete=callback) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) daemon.graceful_shutdown.execute.assert_not_called() - callback.assert_called_once_with(True) + callback.assert_called_once_with(True, "guard_skip") def test_execute_graceful_shutdown_skipped_when_graceful_shutting_down_in_progress(self, chassis): """graceful_shutdown is skipped when STATE_DB device_power_state shows GRACEFUL_SHUTTING_DOWN.""" chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) daemon = self._make_daemon(chassis) daemon.controller._update_host_state(bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN) - daemon.graceful_shutdown.execute = MagicMock(return_value=True) + daemon.controller.write_operation_start(TEST_REQUEST_ID, "active") + daemon.graceful_shutdown.execute = MagicMock() callback = MagicMock() - item = bmcctld.ActionItem(bmcctld.ACTION_GRACEFUL_SHUTDOWN, "dup-grace-event", on_complete=callback) - daemon._execute_action_item(item) + item = bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "dup-grace-event", 3, + on_complete=callback) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) daemon.graceful_shutdown.execute.assert_not_called() - callback.assert_called_once_with(True) + callback.assert_called_once_with(True, "guard_skip") def test_execute_power_on_skipped_when_powering_on_in_progress(self, chassis): """power_on is skipped when STATE_DB device_power_state shows POWERING_ON (already in progress).""" @@ -1163,12 +2879,2682 @@ def test_execute_power_on_skipped_when_powering_on_in_progress(self, chassis): chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) daemon = self._make_daemon(chassis) daemon.controller._update_host_state(bmcctld.SWITCH_HOST_POWERING_ON) - daemon.controller.power_on = MagicMock(return_value=True) + daemon.controller.write_operation_start(TEST_REQUEST_ID, "active") + daemon.controller.power_on = MagicMock() + callback = MagicMock() + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "dup-on-event", 5, + on_complete=callback) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) + daemon.controller.power_on.assert_not_called() + callback.assert_called_once_with(True, "guard_skip") + + def test_p_terminal_powering_on_residue_is_retryable(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller._update_host_state(bmcctld.SWITCH_HOST_POWERING_ON) + daemon.controller.write_operation_result( + bmcctld.OP_RESULT_PREEMPTED, bmcctld.OP_REASON_PREEMPTED) + daemon.controller.power_on = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "retry-after-preempt", 5)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + + daemon.controller.power_on.assert_called_once() + + @pytest.mark.parametrize("oper_status", [ + MockModule.MODULE_STATUS_ONLINE, + MockModule.MODULE_STATUS_OFFLINE, + ]) + def test_failed_power_off_residue_is_not_guarded( + self, oper_status, chassis): + chassis.switch_host.set_oper_status(oper_status) + daemon = self._make_daemon(chassis) + daemon.controller._update_host_state(bmcctld.SWITCH_HOST_POWERING_OFF) + daemon.controller.write_operation_result( + bmcctld.OP_RESULT_POWER_OFF_FAILED, "-") + daemon.controller.power_off = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "retry", 2)) + + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + + daemon.controller.power_off.assert_called_once() + + def test_critical_shutdown_ignores_recorded_transition(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.controller._update_host_state( + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN) + daemon.controller.write_operation_start(TEST_REQUEST_ID, "active") + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "critical", 0) + assert daemon.operation_runner._guard_should_skip(item) is False + + +class TestOperationRunnerConcurrency: + + def _make_daemon(self, chassis): + with patch('sonic_platform.platform.Platform') as platform: + platform.return_value.get_chassis.return_value = chassis + return bmcctld.BmcctldDaemon(bmcctld.SYSLOG_IDENTIFIER) + + def test_p_priority_queue_orders_by_priority_then_arrival(self, chassis): + daemon = self._make_daemon(chassis) + items = [ + bmcctld.ActionItem(bmcctld.ACTION_POWER_ON, "on", 5), + bmcctld.ActionItem(bmcctld.ACTION_POWER_CYCLE, "cycle-a", 4), + bmcctld.ActionItem(bmcctld.ACTION_POWER_CYCLE, "cycle-b", 4), + bmcctld.ActionItem(bmcctld.ACTION_GRACEFUL_SHUTDOWN, "shutdown", 3), + bmcctld.ActionItem(bmcctld.ACTION_POWER_OFF, "critical-off", 0), + ] + for item in items: + daemon.operation_runner.enqueue(item) + ordered = [daemon.action_queue.get_nowait()[2].event_desc + for _ in items] + assert ordered == [ + "critical-off", "shutdown", "cycle-a", "cycle-b", "on"] + with pytest.raises(AttributeError): + items[0].priority = 0 + + def test_q_operation_record_lifecycle_and_release(self, chassis): + daemon = self._make_daemon(chassis) + daemon.controller.power_on = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "record-test", 5) + daemon.operation_runner.enqueue(item) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + start = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert start[bmcctld.FIELD_OP_REQUEST_ID] == operation.request_id + assert start[bmcctld.FIELD_OP_TRIGGER] == "record-test" + assert start[bmcctld.FIELD_OP_RESULT] == "-" + assert start[bmcctld.FIELD_OP_REASON] == "-" + assert operation.thread.name == "bmcctld-op" + assert operation.thread.daemon is True + operation.thread.join() + daemon.operation_runner.process_next(timeout=0) + terminal = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert terminal[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert terminal[bmcctld.FIELD_OP_REASON] == "-" + assert daemon.operation_runner.current is None + assert operation.joined_callbacks == [] + + def test_q_shared_rack_command_row_keeps_fields_through_completion( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.controller.power_off = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + command_table = Table( + daemon._thread_database.connection("STATE_DB"), + bmcctld.RACK_MANAGER_COMMAND_TABLE) + command_table.set("CMD_SHARED", FieldValuePairs([ + (bmcctld.FIELD_COMMAND, bmcctld.CMD_POWER_OFF), + (bmcctld.FIELD_STATUS, bmcctld.CMD_STATUS_PENDING), + ("opaque", "preserve-me"), + ])) + table_factory = bmcctld.swsscommon.Table + + def shared_table(db, table_name): + if table_name == bmcctld.RACK_MANAGER_COMMAND_TABLE: + return command_table + return table_factory(db, table_name) + + with patch('bmcctld.swsscommon.Table', side_effect=shared_table): + command_fvs = dict(command_table.get("CMD_SHARED")[1]) + daemon.event_handler._handle_rack_mgr_command( + "CMD_SHARED", command_fvs) + in_progress = dict(command_table.get("CMD_SHARED")[1]) + assert in_progress[bmcctld.FIELD_STATUS] == \ + bmcctld.CMD_STATUS_IN_PROGRESS + assert bmcctld.FIELD_REQUEST_ID not in in_progress + + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + request_id = operation.request_id + spawned = dict(command_table.get("CMD_SHARED")[1]) + assert spawned[bmcctld.FIELD_REQUEST_ID] == request_id + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + completed = dict(command_table.get("CMD_SHARED")[1]) + assert completed[bmcctld.FIELD_COMMAND] == bmcctld.CMD_POWER_OFF + assert completed[bmcctld.FIELD_STATUS] == bmcctld.CMD_STATUS_DONE + assert completed[bmcctld.FIELD_RESULT] == "SUCCESS" + assert completed[bmcctld.FIELD_REQUEST_ID] == request_id + assert completed["opaque"] == "preserve-me" + + def test_q_graceful_restart_rack_retry_joins_one_operation( + self, chassis): + daemon = self._make_daemon(chassis) + started = threading.Event() + release = threading.Event() + + def blocked_restart(_operation, _factory): + started.set() + assert release.wait(2) + return bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True + + daemon.graceful_shutdown.execute_restart = MagicMock( + side_effect=blocked_restart) + command_table = Table( + daemon._thread_database.connection("STATE_DB"), + bmcctld.RACK_MANAGER_COMMAND_TABLE) + for key in ("RESTART_1", "RESTART_2"): + command_table.set(key, FieldValuePairs([ + (bmcctld.FIELD_COMMAND, bmcctld.CMD_GRACEFUL_RESTART), + (bmcctld.FIELD_STATUS, bmcctld.CMD_STATUS_PENDING), + ])) + table_factory = bmcctld.swsscommon.Table + + def shared_table(db, table_name): + if table_name == bmcctld.RACK_MANAGER_COMMAND_TABLE: + return command_table + return table_factory(db, table_name) + + with patch('bmcctld.swsscommon.Table', side_effect=shared_table): + daemon.event_handler._handle_rack_mgr_command( + "RESTART_1", dict(command_table.get("RESTART_1")[1])) + daemon.operation_runner.process_next(timeout=0) + assert started.wait(1) + operation = daemon.operation_runner.current + request_id = operation.request_id + + daemon.event_handler._handle_rack_mgr_command( + "RESTART_2", dict(command_table.get("RESTART_2")[1])) + daemon.operation_runner.process_next(timeout=0) + assert len(operation.joined_callbacks) == 1 + for key in ("RESTART_1", "RESTART_2"): + in_progress = dict(command_table.get(key)[1]) + assert in_progress[bmcctld.FIELD_STATUS] == \ + bmcctld.CMD_STATUS_IN_PROGRESS + assert in_progress[bmcctld.FIELD_REQUEST_ID] == request_id + + release.set() + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + daemon.graceful_shutdown.execute_restart.assert_called_once() + for key in ("RESTART_1", "RESTART_2"): + completed = dict(command_table.get(key)[1]) + assert completed[bmcctld.FIELD_COMMAND] == \ + bmcctld.CMD_GRACEFUL_RESTART + assert completed[bmcctld.FIELD_STATUS] == \ + bmcctld.CMD_STATUS_DONE + assert completed[bmcctld.FIELD_RESULT] == "SUCCESS" + assert completed[bmcctld.FIELD_REQUEST_ID] == request_id + + def test_s_under_lock_raise_refusal_maps_to_operation_and_callback( + self, chassis): + daemon = self._make_daemon(chassis) + daemon.controller.power_on = MagicMock( + return_value=bmcctld.PowerCallResult.REFUSED_LEAK) + callback = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "late-critical", 5, + on_complete=callback)) + + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_OFF_LEAK_BLOCKED + callback.assert_called_once_with(False, "CRITICAL_LEAK_PRESENT") + + @pytest.mark.parametrize("action,priority", [ + (bmcctld.ACTION_POWER_OFF, 2), + (bmcctld.ACTION_POWER_OFF, 0), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, 3), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, 1), + ]) + @pytest.mark.parametrize("power_fails", [False, True]) + def test_p_identical_request_joins(self, chassis, action, priority, + power_fails): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.policy_reader.get_graceful_shutdown_timeout = MagicMock(return_value=0) + daemon.graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + daemon.operation_runner._event_log = MagicMock() + daemon.operation_runner._record_terminal = MagicMock( + wraps=daemon.operation_runner._record_terminal) + started = threading.Event() + release = threading.Event() + set_admin_state = chassis.switch_host.set_admin_state + + def blocked_power_off(up): + assert up is False + started.set() + assert release.wait(2) + if power_fails: + raise OSError("power-off failed") + set_admin_state(up) + + chassis.switch_host.set_admin_state = MagicMock(side_effect=blocked_power_off) + daemon.event_handler._set_cmd_request_id = MagicMock() + # Critical event producers do not attach callbacks; exercise the runner contract. + first_callback = MagicMock() + second_callback = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + action, "first", priority, + on_complete=first_callback, rack_cmd_key="CMD_1")) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + request_id = operation.request_id + try: + assert started.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + action, "retry", priority, + on_complete=second_callback, rack_cmd_key="CMD_2")) + daemon.operation_runner.process_next(timeout=0) + assert daemon.operation_runner.current is operation + assert not operation.cancel.is_set() + assert operation.joined_callbacks == [(second_callback, "CMD_2")] + first_callback.assert_not_called() + second_callback.assert_not_called() + daemon.operation_runner._event_log.log_notice.assert_any_call( + "JOIN action={} request_id={}".format(action, request_id)) + finally: + release.set() + operation.thread.join(2) + assert not operation.thread.is_alive() + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.process_next(timeout=0) + chassis.switch_host.set_admin_state.assert_called_once_with(False) + expected_result = (bmcctld.OP_RESULT_POWER_OFF_FAILED if power_fails else + bmcctld.OP_RESULT_SUCCESS if action == bmcctld.ACTION_POWER_OFF else + bmcctld.OP_RESULT_SUCCESS_FORCED) + first_callback.assert_called_once_with(not power_fails, expected_result) + second_callback.assert_called_once_with(not power_fails, expected_result) + assert daemon.operation_runner.current is None + daemon.operation_runner._record_terminal.assert_called_once() + assert daemon.event_handler._set_cmd_request_id.call_args_list == [ + call("CMD_1", request_id), + call("CMD_2", request_id), + ] + + def test_p_equal_or_lower_priority_is_refused_busy(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + started = threading.Event() + release = threading.Event() + def blocked_power_off(_operation, _cancel): + started.set() + assert release.wait(2) + return bmcctld.PowerCallResult.CONFIRMED + daemon.controller.power_off = MagicMock(side_effect=blocked_power_off) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "running", 2)) + daemon.operation_runner.process_next(timeout=0) + assert started.wait(1) + running_id = daemon.operation_runner.current.request_id + callback = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "lower", 5, on_complete=callback)) + daemon.operation_runner.process_next(timeout=0) + callback.assert_called_once_with(False, "BUSY") + assert daemon.operation_runner.current.request_id == running_id + release.set() + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + @pytest.mark.parametrize("action,priority", [ + (bmcctld.ACTION_POWER_OFF, 0), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, 1), + ]) + @pytest.mark.parametrize("rack_first", [False, True]) + def test_p_critical_event_sources_join_identical_action( + self, chassis, action, priority, rack_first): + daemon = self._make_daemon(chassis) + daemon.policy_reader.get_leak_control_policy = MagicMock(return_value={ + "system_leak_policy": "enabled", + "system_critical_leak_action": action, + "rack_mgr_leak_policy": "enabled", + "rack_mgr_critical_alert_action": action, + }) + events = [ + (daemon.event_handler._handle_system_leak, + bmcctld.SYSTEM_LEAK_STATUS_KEY, + {bmcctld.FIELD_DEVICE_LEAK_STATUS: bmcctld.SYSTEM_LEAK_CRITICAL}), + (daemon.event_handler._handle_rack_mgr_alert, "Rack_level_leak", + {bmcctld.FIELD_SEVERITY: bmcctld.ALERT_SEVERITY_CRITICAL}), + ] + if rack_first: + events.reverse() + handler, key, fields = events[0] + handler(key, fields) + first_item = _dequeue_item(daemon.action_queue) + assert first_item.priority == priority + assert first_item.on_complete is None + assert first_item.rack_cmd_key is None + operation = bmcctld.Operation( + first_item, TEST_REQUEST_ID, + bmcctld.OperationRunner.INITIAL_STAGES[action]) + operation.thread = MagicMock() + operation.thread.is_alive.return_value = True + daemon.operation_runner.current = operation + daemon.operation_runner._event_log = MagicMock() + daemon.operation_runner._spawn = MagicMock() + daemon.event_handler._set_cmd_request_id = MagicMock() + handler, key, fields = events[1] + handler(key, fields) + + daemon.operation_runner.process_next(timeout=0) + + assert daemon.operation_runner.current is operation + assert not operation.cancel.is_set() + assert operation.joined_callbacks == [] + assert daemon.action_queue.empty() + daemon.operation_runner._spawn.assert_not_called() + daemon.event_handler._set_cmd_request_id.assert_not_called() + daemon.operation_runner._event_log.log_notice.assert_called_once_with( + "JOIN action={} request_id={}".format(action, TEST_REQUEST_ID)) + + @pytest.mark.parametrize("running_action,running_priority,new_action,new_priority", [ + (bmcctld.ACTION_POWER_OFF, 0, bmcctld.ACTION_GRACEFUL_SHUTDOWN, 1), + (bmcctld.ACTION_GRACEFUL_RESTART, 4, bmcctld.ACTION_POWER_CYCLE, 4), + (bmcctld.ACTION_POWER_CYCLE, 4, bmcctld.ACTION_GRACEFUL_RESTART, 4), + ]) + def test_p_conflicting_actions_remain_busy( + self, chassis, running_action, running_priority, new_action, new_priority): + daemon = self._make_daemon(chassis) + operation = _make_operation(running_action, priority=running_priority) + operation.thread = MagicMock() + operation.thread.is_alive.return_value = True + daemon.operation_runner.current = operation + daemon.operation_runner._event_log = MagicMock() + callback = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + new_action, "conflict", new_priority, on_complete=callback)) + + daemon.operation_runner.process_next(timeout=0) + + callback.assert_called_once_with(False, "BUSY") + assert daemon.operation_runner.current is operation + assert not operation.cancel.is_set() + assert operation.joined_callbacks == [] + assert daemon.action_queue.empty() + daemon.operation_runner._event_log.log_notice.assert_called_once_with( + "REFUSED_BUSY action={} running_request_id={}".format( + new_action, TEST_REQUEST_ID)) + + def test_p_critical_same_action_joins_and_raises_rank(self, chassis): + daemon = self._make_daemon(chassis) + operation = _make_operation( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, priority=3) + operation.thread = MagicMock() + operation.thread.is_alive.return_value = True + daemon.operation_runner.current = operation + daemon.operation_runner._event_log = MagicMock() + daemon.operation_runner._spawn = MagicMock() + daemon.operation_runner._displace_and_requeue = MagicMock() + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "CRITICAL_SYSTEM_LEAK", 1)) + daemon.operation_runner.process_next(timeout=0) + + assert daemon.operation_runner.current is operation + assert not operation.cancel.is_set() + assert operation.priority == 1 + daemon.operation_runner._spawn.assert_not_called() + daemon.operation_runner._displace_and_requeue.assert_not_called() + daemon.operation_runner._event_log.log_notice.assert_has_calls([ + call("JOIN raised priority request_id={} priority=1".format( + TEST_REQUEST_ID)), + call("JOIN action={} request_id={}".format( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, TEST_REQUEST_ID)), + ]) + + # An ordinary power-off no longer outranks the joined operation. callback = MagicMock() - item = bmcctld.ActionItem(bmcctld.ACTION_POWER_ON, "dup-on-event", on_complete=callback) - daemon._execute_action_item(item) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "RACK_MGR_CMD:POWER_OFF", 2, + on_complete=callback)) + daemon.operation_runner.process_next(timeout=0) + callback.assert_called_once_with(False, "BUSY") + daemon.operation_runner._displace_and_requeue.assert_not_called() + + # A critical power-off still does. + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "CRITICAL_SYSTEM_LEAK", 0)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner._displace_and_requeue.assert_called_once() + + def test_p_critical_graceful_joins_running_graceful_without_second_halt( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=True) + daemon.policy_reader.get_graceful_shutdown_timeout = MagicMock( + return_value=120) + halt_sent = threading.Event() + release = threading.Event() + requester = MagicMock() + + def send_halt(_message, timeout_secs): + halt_sent.set() + + def poll_status(timeout_secs): + assert release.wait(2) + return _report("Halt reboot completed | [bmc-req:{}]".format( + daemon.operation_runner.current.request_id)) + + requester.send_halt.side_effect = send_halt + requester.poll_status.side_effect = poll_status + daemon.gnoi_requester_factory = MagicMock(return_value=requester) + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "RACK_MGR_CMD:GRACEFUL_SHUT", 3)) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert halt_sent.wait(2) + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "CRITICAL_SYSTEM_LEAK", 1)) + daemon.operation_runner.process_next(timeout=0) + assert daemon.operation_runner.current is operation + assert not operation.cancel.is_set() + assert operation.priority == 1 + + release.set() + operation.thread.join(2) + assert not operation.thread.is_alive() + assert operation.outcome == ( + bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True) + daemon.operation_runner.process_next(timeout=0) + + requester.send_halt.assert_called_once() + daemon.gnoi_requester_factory.assert_called_once() + assert daemon.operation_runner.current is None + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_REQUEST_ID] == operation.request_id + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_SUCCESS_GRACEFUL + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_OFFLINE + + def test_p_critical_power_off_joins_running_ordinary_power_off( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + started = threading.Event() + release = threading.Event() + set_admin_state = chassis.switch_host.set_admin_state + + def blocked_power_off(up): + assert up is False + started.set() + assert release.wait(2) + set_admin_state(up) + + chassis.switch_host.set_admin_state = MagicMock( + side_effect=blocked_power_off) + rack_callback = MagicMock() + leak_callback = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "RACK_MGR_CMD:POWER_OFF", 2, + on_complete=rack_callback, rack_cmd_key="CMD_1")) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert started.wait(2) + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "CRITICAL_SYSTEM_LEAK", 0, + on_complete=leak_callback)) + daemon.operation_runner.process_next(timeout=0) + assert daemon.operation_runner.current is operation + assert not operation.cancel.is_set() + assert operation.priority == 0 + + release.set() + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + chassis.switch_host.set_admin_state.assert_called_once_with(False) + rack_callback.assert_called_once_with(True, bmcctld.OP_RESULT_SUCCESS) + leak_callback.assert_called_once_with(True, bmcctld.OP_RESULT_SUCCESS) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_REQUEST_ID] == operation.request_id + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + + @pytest.mark.parametrize("running_priority,new_action,new_priority", [ + (3, bmcctld.ACTION_POWER_OFF, 0), + (1, bmcctld.ACTION_POWER_OFF, 0), + ]) + def test_p_higher_priority_cancels_records_then_requeues( + self, chassis, running_priority, new_action, new_priority): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + started = threading.Event() + def graceful_wait(operation, _factory): + started.set() + assert operation.cancel.wait(2) + return (bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, False) + daemon.graceful_shutdown.execute = MagicMock(side_effect=graceful_wait) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "running", running_priority)) + daemon.operation_runner.process_next(timeout=0) + assert started.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + new_action, "critical", new_priority)) + daemon.operation_runner.process_next(timeout=0) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_PREEMPTED + assert state[bmcctld.FIELD_OP_REASON] == bmcctld.OP_REASON_PREEMPTED + assert daemon.operation_runner.current is None + assert _dequeue_item(daemon.action_queue).event_desc == "critical" + + def test_p_critical_off_preempts_restart_pause_and_is_guard_resolved( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.policy_reader.get_graceful_shutdown_timeout = \ + MagicMock(return_value=0) + daemon.graceful_shutdown._is_graceful_qualified = \ + MagicMock(return_value=True) + original_set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=original_set_admin_state) + daemon.controller.power_on = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_RESTART, "restart", 4)) + daemon.operation_runner.process_next(timeout=0) + restart_operation = daemon.operation_runner.current + + deadline = time.monotonic() + 1 + while restart_operation.stage != bmcctld.STAGE_PAUSE and \ + time.monotonic() < deadline: + time.sleep(0.001) + assert restart_operation.stage == bmcctld.STAGE_PAUSE + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "critical-off", 0)) + daemon.operation_runner.process_next(timeout=0) + + assert restart_operation.outcome == ( + bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_TIMEOUT_ZERO, + False, + ) + assert daemon.operation_runner.current is None daemon.controller.power_on.assert_not_called() - callback.assert_called_once_with(True) + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_OFFLINE + + daemon.operation_runner.process_next(timeout=0) + + assert daemon.operation_runner.current is None + assert daemon.action_queue.empty() + assert chassis.switch_host.set_admin_state.call_args_list == [ + call(False)] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_DEVICE_STATUS] == \ + bmcctld.SWITCH_HOST_OFFLINE + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_SUCCESS + + def test_admin_down_offline_completes_through_common_guard(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller._update_host_state( + bmcctld.POWER_STATE_OFF, bmcctld.SWITCH_HOST_OFFLINE) + daemon.controller.power_off = MagicMock() + daemon.operation_runner._new_request_id = MagicMock( + return_value=TEST_UUID4) + + daemon.event_handler._handle_chassis_module( + bmcctld.SWITCH_HOST_MODULE_KEY, + {bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_DOWN}, + ) + + assert daemon.operation_runner.process_next(timeout=0) is True + assert daemon.operation_runner.current is None + assert daemon.action_queue.empty() + daemon.controller.power_off.assert_not_called() + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_REQUEST_ID] == TEST_UUID4 + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_SUCCESS_FORCED + assert state[bmcctld.FIELD_OP_REASON] == \ + bmcctld.OP_REASON_ALREADY_OFF + + @pytest.mark.parametrize( + "timeout, graceful, expected_reason", + [ + (0, False, bmcctld.OP_REASON_TIMEOUT_ZERO), + (10, True, "-"), + ], + ids=["timeout-zero", "tagged-graceful"], + ) + def test_admin_down_preempts_restart_while_host_is_off( + self, timeout, graceful, expected_reason, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.policy_reader.get_graceful_shutdown_timeout = MagicMock( + return_value=timeout) + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=True) + requester = MagicMock() + requester.poll_status.return_value = _report( + "done [bmc-req:{}]".format(TEST_REQUEST_ID)) + daemon.gnoi_requester_factory = MagicMock(return_value=requester) + daemon.operation_runner._new_request_id = MagicMock( + return_value=TEST_REQUEST_ID) + original_set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=original_set_admin_state) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_RESTART, "restart", 4)) + daemon.operation_runner.process_next(timeout=0) + restart_operation = daemon.operation_runner.current + + deadline = time.monotonic() + 1 + while restart_operation.stage != bmcctld.STAGE_PAUSE and \ + time.monotonic() < deadline: + time.sleep(0.001) + reached_pause = restart_operation.stage == bmcctld.STAGE_PAUSE + if not reached_pause: + restart_operation.cancel.set() + restart_operation.thread.join(2) + assert reached_pause + assert restart_operation.leg_graceful is graceful + + daemon.event_handler._handle_chassis_module( + bmcctld.SWITCH_HOST_MODULE_KEY, + {bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_DOWN}, + ) + admitted = not daemon.action_queue.empty() + if admitted: + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.process_next(timeout=0) + else: + restart_operation.cancel.set() + restart_operation.thread.join(2) + + assert admitted + assert restart_operation.outcome == ( + bmcctld.OP_RESULT_PREEMPTED, expected_reason, False) + assert daemon.operation_runner.current is None + assert daemon.action_queue.empty() + assert chassis.switch_host.set_admin_state.call_args_list == [ + call(False)] + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_OFFLINE + + def test_admin_down_preempts_power_on_verification(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller._update_host_state( + bmcctld.SWITCH_HOST_POWERING_ON, + bmcctld.SWITCH_HOST_OFFLINE) + verifying = threading.Event() + + def power_on_verification(operation, cancel): + operation.stage = bmcctld.STAGE_POWER_ON_ISSUED + chassis.switch_host.set_oper_status( + MockModule.MODULE_STATUS_ONLINE) + verifying.set() + assert cancel.wait(2) + return bmcctld.PowerCallResult.CANCELLED + + def confirmed_power_off(_operation, _cancel): + chassis.switch_host.set_oper_status( + MockModule.MODULE_STATUS_OFFLINE) + return bmcctld.PowerCallResult.CONFIRMED + + daemon.controller.power_on = MagicMock( + side_effect=power_on_verification) + daemon.controller.power_off = MagicMock( + side_effect=confirmed_power_off) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "power-on", 5)) + daemon.operation_runner.process_next(timeout=0) + power_on_operation = daemon.operation_runner.current + assert verifying.wait(1) + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_ONLINE + + daemon.event_handler._handle_chassis_module( + bmcctld.SWITCH_HOST_MODULE_KEY, + {bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_DOWN}, + ) + admitted = not daemon.action_queue.empty() + if admitted: + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.process_next(timeout=0) + shutdown_operation = daemon.operation_runner.current + shutdown_operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + else: + power_on_operation.cancel.set() + power_on_operation.thread.join(2) + + assert admitted + assert power_on_operation.outcome == ( + bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, + False, + ) + assert daemon.operation_runner.current is None + assert daemon.action_queue.empty() + daemon.controller.power_off.assert_called_once() + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_OFFLINE + + def test_admin_up_during_shutdown_remains_lower_priority(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + started = threading.Event() + release = threading.Event() + + def blocked_shutdown(_operation, _factory): + started.set() + assert release.wait(2) + return (bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True) + + daemon.graceful_shutdown.execute = MagicMock( + side_effect=blocked_shutdown) + daemon.controller.write_operation_start = MagicMock( + wraps=daemon.controller.write_operation_start) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "shutdown", 3)) + daemon.operation_runner.process_next(timeout=0) + shutdown_operation = daemon.operation_runner.current + assert started.wait(1) + daemon.controller._update_host_state( + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN, + bmcctld.SWITCH_HOST_OFFLINE) + daemon.event_handler.critical_event_checker.has_any_critical_event = \ + MagicMock(return_value=False) + + daemon.event_handler._handle_chassis_module( + bmcctld.SWITCH_HOST_MODULE_KEY, + {bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_UP}, + ) + assert not daemon.action_queue.empty() + assert daemon.operation_runner.process_next(timeout=0) is True + + assert daemon.operation_runner.current is shutdown_operation + assert daemon.action_queue.empty() + assert daemon.controller.write_operation_start.call_count == 1 + release.set() + shutdown_operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + def test_stop_reaps_worker_that_finishes_during_join(self, chassis): + daemon = self._make_daemon(chassis) + first_callback = MagicMock() + second_callback = MagicMock() + operation = _make_operation( + bmcctld.ACTION_POWER_ON, callback=first_callback) + operation.joined_callbacks = [(second_callback, None)] + + def finish_after_cancel(): + assert operation.cancel.wait(2) + operation.outcome = ( + bmcctld.OP_RESULT_PREEMPTED, "-", False) + + operation.thread = threading.Thread( + target=finish_after_cancel, daemon=True) + daemon.controller.write_operation_start( + operation.request_id, operation.item.event_desc) + daemon.operation_runner.current = operation + operation.thread.start() + + daemon.operation_runner.stop() + + # Cancelled by the stop, not displaced: recorded as abandoned. + assert daemon.operation_runner.current is None + first_callback.assert_called_once_with( + False, bmcctld.OP_RESULT_ABANDONED) + second_callback.assert_called_once_with( + False, bmcctld.OP_RESULT_ABANDONED) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_ABANDONED + assert state[bmcctld.FIELD_OP_REASON] == "-" + + daemon.operation_runner.stop() + first_callback.assert_called_once() + second_callback.assert_called_once() + + def test_stop_keeps_outcome_of_worker_that_already_finished(self, chassis): + daemon = self._make_daemon(chassis) + callback = MagicMock() + operation = _make_operation( + bmcctld.ACTION_POWER_ON, callback=callback) + operation.outcome = (bmcctld.OP_RESULT_SUCCESS, "-", True) + operation.thread = MagicMock() + operation.thread.is_alive.return_value = False + daemon.controller.write_operation_start( + operation.request_id, operation.item.event_desc) + daemon.operation_runner.current = operation + + daemon.operation_runner.stop() + + assert not operation.cancel.is_set() + callback.assert_called_once_with(True, bmcctld.OP_RESULT_SUCCESS) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + + def test_stop_after_displacement_keeps_preempted(self, chassis): + daemon = self._make_daemon(chassis) + callback = MagicMock() + operation = _make_operation( + bmcctld.ACTION_POWER_ON, callback=callback) + # A displacement already cancelled the worker; the stop only joins it. + operation.cancel.set() + release = threading.Event() + + def finish(): + assert release.wait(2) + operation.outcome = (bmcctld.OP_RESULT_PREEMPTED, "-", False) + + operation.thread = threading.Thread(target=finish, daemon=True) + daemon.controller.write_operation_start( + operation.request_id, operation.item.event_desc) + daemon.operation_runner.current = operation + operation.thread.start() + real_join = operation.thread.join + + def release_then_join(timeout=None): + release.set() + real_join(timeout) + + operation.thread.join = release_then_join + + daemon.operation_runner.stop() + + assert operation.cancelled_by_stop is False + callback.assert_called_once_with(False, bmcctld.OP_RESULT_PREEMPTED) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_PREEMPTED + + @pytest.mark.parametrize("stop_before_join", [False, True]) + @pytest.mark.parametrize("outcome", [ + None, (bmcctld.OP_RESULT_SUCCESS, "-", True), + ]) + def test_displacement_stop_preserves_live_worker_and_queue( + self, chassis, stop_before_join, outcome): + daemon = self._make_daemon(chassis) + runner = daemon.operation_runner + callback, joined_callback = MagicMock(), MagicMock() + operation = _make_operation( + bmcctld.ACTION_POWER_ON, callback=callback) + operation.joined_callbacks.append((joined_callback, None)) + operation.outcome = outcome + operation.thread = MagicMock() + operation.thread.is_alive.return_value = True + runner.current = operation + runner._spawn = MagicMock() + daemon.controller.write_operation_start( + operation.request_id, operation.item.event_desc) + daemon.controller.write_operation_result = MagicMock() + successor = bmcctld.ActionItem(bmcctld.ACTION_POWER_OFF, "stop", 2) + entry = (successor.priority, 42, successor) + daemon.action_queue.put(entry) + + def stop_during_join(timeout): + assert operation.cancel.is_set() + assert not daemon.stop_event.is_set(), "joined again after daemon stop" + daemon.stop_event.set() + + operation.thread.join.side_effect = stop_during_join + if stop_before_join: + daemon.stop_event.set() + + assert runner.process_next(timeout=0) is True + + assert operation.thread.join.call_args_list == ( + [] if stop_before_join else [call(timeout=1)]) + assert operation.cancel.is_set() + assert runner.current is operation + assert operation.outcome is outcome + assert daemon.action_queue.get_nowait() is entry + assert daemon.action_queue.empty() + runner._spawn.assert_not_called() + daemon.controller.write_operation_result.assert_not_called() + callback.assert_not_called() + joined_callback.assert_not_called() + state = dict(daemon.controller.host_state_table.get(bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == "-" + + def test_displacement_waits_for_worker_with_periodic_warnings( + self, chassis, monkeypatch): + daemon = self._make_daemon(chassis) + runner = daemon.operation_runner + callback = MagicMock() + operation = _make_operation( + bmcctld.ACTION_POWER_ON, callback=callback) + operation.thread = MagicMock() + operation.thread.is_alive.return_value = True + runner.current = operation + runner._spawn = MagicMock() + daemon.controller.write_operation_result = MagicMock() + successor = bmcctld.ActionItem(bmcctld.ACTION_POWER_OFF, "next", 2) + entry = (successor.priority, 42, successor) + daemon.action_queue.put(entry) + now = 0 + warnings = [] + monkeypatch.setattr(bmcctld.time, "monotonic", lambda: now) + runner.log_warning = MagicMock(side_effect=lambda message: warnings.append(now)) + + def join_worker(timeout): + nonlocal now + assert timeout == 1 + assert operation.cancel.is_set() + runner._spawn.assert_not_called() + daemon.controller.write_operation_result.assert_not_called() + callback.assert_not_called() + now += timeout + if now == 61: + operation.outcome = ( + bmcctld.OP_RESULT_PREEMPTED, bmcctld.OP_REASON_PREEMPTED, False) + operation.thread.is_alive.return_value = False + + operation.thread.join.side_effect = join_worker + + assert runner.process_next(timeout=0) is True + + assert operation.thread.join.call_args_list == [call(timeout=1)] * 61 + assert warnings == [30, 60] + assert runner.current is None + assert daemon.action_queue.get_nowait() is entry + runner._spawn.assert_not_called() + daemon.controller.write_operation_result.assert_called_once_with( + bmcctld.OP_RESULT_PREEMPTED, bmcctld.OP_REASON_PREEMPTED) + callback.assert_called_once_with(False, bmcctld.OP_RESULT_PREEMPTED) + + def test_p_completed_worker_is_reaped_after_blocking_get(self, chassis): + daemon = self._make_daemon(chassis) + release = threading.Event() + operation = _make_operation(bmcctld.ACTION_POWER_OFF) + def finish_old(): + release.wait(2) + operation.outcome = (bmcctld.OP_RESULT_SUCCESS, "-", True) + operation.thread = threading.Thread(target=finish_old, daemon=True) + daemon.operation_runner.current = operation + operation.thread.start() + recorded = [] + original_record = daemon.controller.write_operation_result + def record(result, reason): + recorded.append((result, reason)) + original_record(result, reason) + daemon.controller.write_operation_result = MagicMock(side_effect=record) + daemon.controller.power_on = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + + processor = threading.Thread( + target=daemon.operation_runner.process_next, + kwargs={"timeout": 2}) + processor.start() + time.sleep(0.05) + release.set() + operation.thread.join(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "new", 5)) + processor.join(2) + assert processor.is_alive() is False + assert recorded[0] == (bmcctld.OP_RESULT_SUCCESS, "-") + assert all(result != bmcctld.OP_RESULT_PREEMPTED + for result, _reason in recorded) + assert daemon.operation_runner.current.item.event_desc == "new" + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + def test_p_completed_before_cancel_keeps_truthful_outcome(self, chassis): + daemon = self._make_daemon(chassis) + outcome_set = threading.Event() + operation = _make_operation(bmcctld.ACTION_POWER_ON, priority=5) + + def complete_then_linger(): + operation.outcome = (bmcctld.OP_RESULT_SUCCESS, "-", True) + outcome_set.set() + assert operation.cancel.wait(2) + + operation.thread = threading.Thread( + target=complete_then_linger, daemon=True) + daemon.controller.write_operation_start( + operation.request_id, operation.item.event_desc) + daemon.operation_runner.current = operation + operation.thread.start() + assert outcome_set.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "higher-priority", 2)) + + daemon.operation_runner.process_next(timeout=0) + + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert daemon.operation_runner.current is None + assert _dequeue_item(daemon.action_queue).event_desc == \ + "higher-priority" + + def test_p_displacement_preserves_entry_and_new_higher_priority_wins( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + started = threading.Event() + release = threading.Event() + def blocked_cycle(operation, _cancel): + started.set() + assert release.wait(2) + if operation.cancel.is_set(): + return bmcctld.PowerCallResult.CANCELLED + return bmcctld.PowerCallResult.CONFIRMED + daemon.controller.power_cycle = MagicMock(side_effect=blocked_cycle) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_CYCLE, "running-cycle", 4)) + daemon.operation_runner.process_next(timeout=0) + assert started.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "requeued", 3)) + displacer = threading.Thread( + target=daemon.operation_runner.process_next, + kwargs={"timeout": 0}) + displacer.start() + assert daemon.operation_runner.current.cancel.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "late-critical", 0)) + release.set() + displacer.join(2) + assert displacer.is_alive() is False + entries = [daemon.action_queue.get_nowait(), daemon.action_queue.get_nowait()] + assert [entry[2].event_desc for entry in entries] == [ + "late-critical", "requeued"] + assert entries[1][1] < entries[0][1] + + def test_p_requeued_successor_runs_the_common_guard(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + started = threading.Event() + + def checkpoint_passed_power_off(operation, _factory): + started.set() + assert operation.cancel.wait(2) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + # Recorded power-off plus PREEMPTED, as a cancelled restart pause leaves it. + daemon.controller._update_host_state(bmcctld.POWER_STATE_OFF) + return (bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, False) + + daemon.graceful_shutdown.execute = MagicMock( + side_effect=checkpoint_passed_power_off) + daemon.controller.power_off = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "running", 3)) + daemon.operation_runner.process_next(timeout=0) + assert started.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "critical-successor", 0)) + + daemon.operation_runner.process_next(timeout=0) + assert daemon.operation_runner.current is None + daemon.operation_runner.process_next(timeout=0) + + daemon.controller.power_off.assert_not_called() + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + + def test_q_successor_after_cancelled_off_verify_issues_power_off( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + verifying = threading.Event() + set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=set_admin_state) + + def cancelled_verify(_expected, _timeout, _context, cancel=None): + if verifying.is_set(): + return bmcctld.PowerCallResult.CONFIRMED + verifying.set() + assert cancel.wait(2) + return bmcctld.PowerCallResult.CANCELLED + + daemon.controller._verify_oper_status = MagicMock( + side_effect=cancelled_verify) + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=False) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "running", 3)) + daemon.operation_runner.process_next(timeout=0) + assert verifying.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "critical-successor", 0)) + + daemon.operation_runner.process_next(timeout=0) + assert daemon.operation_runner.current is None + preempted = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert preempted[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_OFF + assert preempted[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_PREEMPTED + daemon.operation_runner.process_next(timeout=0) + successor = daemon.operation_runner.current + assert successor is not None + successor.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + # The cancelled removal was never confirmed; the successor calls again. + assert chassis.switch_host.set_admin_state.call_args_list == [ + call(False), call(False)] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_DEVICE_STATUS] == \ + bmcctld.SWITCH_HOST_OFFLINE + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert state[bmcctld.FIELD_OP_TRIGGER] == "critical-successor" + + def test_s_startup_verify_is_displaceable(self, chassis): + daemon = self._make_daemon(chassis) + verifying = threading.Event() + + def startup_verify(operation, cancel): + operation.stage = bmcctld.STAGE_POWER_ON_ISSUED + verifying.set() + assert cancel.wait(2) + return bmcctld.PowerCallResult.CANCELLED + + daemon.controller.power_on = MagicMock(side_effect=startup_verify) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "STARTUP", 5, startup=True)) + daemon.operation_runner.process_next(timeout=0) + assert verifying.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "critical", 0)) + + daemon.operation_runner.process_next(timeout=0) + + assert daemon.operation_runner.current is None + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_PREEMPTED + + def test_s_startup_raise_dropped_after_admitted_power_removal( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + daemon.operation_runner._event_log = MagicMock() + callback = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "RACK_MGR_CMD:POWER_OFF", 2, + on_complete=callback, rack_cmd_key="CMD_1")) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "STARTUP", 5, startup=True)) + + daemon.operation_runner.process_next(timeout=0) + callback.assert_called_once_with(True, "guard_skip") + assert daemon.operation_runner.process_next(timeout=0) is True + + assert daemon.operation_runner.current is None + assert daemon.action_queue.empty() + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_OFFLINE + daemon.operation_runner._event_log.log_notice.assert_any_call( + "STARTUP power_on dropped: a power-removing request was " + "admitted first") + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_TRIGGER] == "RACK_MGR_CMD:POWER_OFF" + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.POWER_STATE_OFF + + # Only the automatic raise yields; an explicit power-on still runs. + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "RACK_MGR_CMD:POWER_ON", 5)) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert operation is not None + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_ONLINE + + def test_s_startup_raise_runs_without_prior_power_removal(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "STARTUP", 5, startup=True)) + + daemon.operation_runner.process_next(timeout=0) + + operation = daemon.operation_runner.current + assert operation is not None + assert operation.item.event_desc == "STARTUP" + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_ONLINE + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_TRIGGER] == "STARTUP" + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + + def test_s_startup_raise_admitted_first_is_displaced_by_power_removal( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=set_admin_state) + verifying = threading.Event() + + def verify(expected, _timeout, _context, cancel=None): + if expected == bmcctld.SWITCH_HOST_ONLINE: + verifying.set() + assert cancel.wait(2) + return bmcctld.PowerCallResult.CANCELLED + return bmcctld.PowerCallResult.CONFIRMED + + daemon.controller._verify_oper_status = MagicMock(side_effect=verify) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "STARTUP", 5, startup=True)) + daemon.operation_runner.process_next(timeout=0) + assert verifying.wait(1) + callback = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "RACK_MGR_CMD:POWER_OFF", 2, + on_complete=callback, rack_cmd_key="CMD_1")) + + daemon.operation_runner.process_next(timeout=0) + assert daemon.operation_runner.current is None + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert operation is not None + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + callback.assert_called_once_with(True, bmcctld.OP_RESULT_SUCCESS) + assert chassis.switch_host.set_admin_state.call_args_list == [ + call(True), call(False)] + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_OFFLINE + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_TRIGGER] == "RACK_MGR_CMD:POWER_OFF" + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.POWER_STATE_OFF + + def test_p_preempted_power_on_cannot_absorb_off_successor(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + raise_issued = threading.Event() + admin_calls = [] + + def delayed_admin_state(admin_up): + admin_calls.append(admin_up) + if admin_up: + raise_issued.set() + + chassis.switch_host.set_admin_state = delayed_admin_state + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "STARTUP", 5)) + daemon.operation_runner.process_next(timeout=0) + assert raise_issued.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "critical", 0)) + + daemon.operation_runner.process_next(timeout=0) + preempted = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert preempted[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_ON + assert preempted[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_PREEMPTED + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert admin_calls == [True, False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + + def test_p_preempted_power_cycle_cannot_absorb_off_successor(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + cycle_issued = threading.Event() + admin_calls = [] + + def delayed_cycle(): + cycle_issued.set() + + def record_admin_state(admin_up): + admin_calls.append(admin_up) + + chassis.switch_host.do_power_cycle = delayed_cycle + chassis.switch_host.set_admin_state = record_admin_state + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_CYCLE, "cycle", 4)) + daemon.operation_runner.process_next(timeout=0) + assert cycle_issued.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "critical", 0)) + + daemon.operation_runner.process_next(timeout=0) + preempted = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert preempted[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWER_CYCLING + assert preempted[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_PREEMPTED + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert admin_calls == [False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + + def test_p_failed_off_after_preempted_raise_remains_retryable(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + raise_issued = threading.Event() + admin_calls = [] + + def fail_first_power_off(admin_up): + admin_calls.append(admin_up) + if admin_up: + raise_issued.set() + elif admin_calls.count(False) == 1: + raise RuntimeError("first power-off failed") + + chassis.switch_host.set_admin_state = fail_first_power_off + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "STARTUP", 5)) + daemon.operation_runner.process_next(timeout=0) + assert raise_issued.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "critical", 0)) + + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + failed = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert failed[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_OFF + assert failed[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_OFF_FAILED + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "retry", 2)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert admin_calls == [True, False, False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + + def test_p_preempted_graceful_after_preempted_raise_cannot_absorb_off_successor( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + raise_issued = threading.Event() + in_handshake = threading.Event() + admin_calls = [] + + def accepted_raise(admin_up): + # Rail command accepted; status not up yet. + admin_calls.append(admin_up) + if admin_up: + raise_issued.set() + + def qualified_until_cancelled(_cert_paths): + in_handshake.set() + assert daemon.operation_runner.current.cancel.wait(2) + return True + + chassis.switch_host.set_admin_state = accepted_raise + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + side_effect=qualified_until_cancelled) + daemon.policy_reader.get_graceful_shutdown_timeout = MagicMock( + return_value=120) + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "CHASSIS_MODULE:ADMIN_UP", 5)) + daemon.operation_runner.process_next(timeout=0) + assert raise_issued.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "RACK_MGR_CMD:GRACEFUL_SHUT", 3)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.process_next(timeout=0) + assert in_handshake.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "CRITICAL_SYSTEM_LEAK", 0)) + + daemon.operation_runner.process_next(timeout=0) + preempted = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert preempted[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN + assert preempted[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_PREEMPTED + daemon.operation_runner.process_next(timeout=0) + successor = daemon.operation_runner.current + assert successor is not None + successor.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert admin_calls == [True, False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_TRIGGER] == "CRITICAL_SYSTEM_LEAK" + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + + @pytest.mark.parametrize("successor,priority", [ + (bmcctld.ACTION_POWER_OFF, 0), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, 1), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, 3), + ]) + def test_p_unconfirmed_raise_cannot_absorb_off_successor( + self, chassis, monkeypatch, successor, priority): + monkeypatch.setattr(bmcctld, "POWER_VERIFY_TIMEOUT_SECS", 0) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=False) + admin_calls = [] + chassis.switch_host.set_admin_state = admin_calls.append + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "STARTUP", 5)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + failed = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert failed[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_ON + assert failed[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_ON_FAILED + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + successor, "successor", priority)) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert operation is not None + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert admin_calls == [True, False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_OP_RESULT] in ( + bmcctld.OP_RESULT_SUCCESS, bmcctld.OP_RESULT_SUCCESS_FORCED) + + @pytest.mark.parametrize("raise_action", [ + bmcctld.ACTION_POWER_ON, bmcctld.ACTION_POWER_CYCLE, + ]) + def test_p_failed_issued_raise_cannot_absorb_off_successor( + self, chassis, raise_action): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + calls = [] + chassis.switch_host.set_admin_state = lambda up: calls.append(up) + chassis.switch_host.do_power_cycle = lambda: calls.append("cycle") + daemon.controller._verify_oper_status = MagicMock(side_effect=[ + RuntimeError("status read failed"), + bmcctld.PowerCallResult.CONFIRMED, + ]) + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + raise_action, "raise", bmcctld.action_priority(raise_action))) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + failed = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + # The raise was issued before the failure, so the transitional label + # is kept: power is not known to be off. + assert failed[bmcctld.FIELD_DEVICE_POWER_STATE] == ( + bmcctld.SWITCH_HOST_POWERING_ON + if raise_action == bmcctld.ACTION_POWER_ON + else bmcctld.SWITCH_HOST_POWER_CYCLING) + assert failed[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_ON_FAILED + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "CRITICAL_SYSTEM_LEAK", 0)) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert operation is not None + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert calls == [True if raise_action == bmcctld.ACTION_POWER_ON + else "cycle", False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + + def test_p_preempted_off_before_issue_cannot_absorb_off_successor( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=False) + raise_issued = threading.Event() + in_window = threading.Event() + admin_calls = [] + graceful_reads = [] + real_get_oper_status = chassis.switch_host.get_oper_status + + def accepted_raise(admin_up): + # Rail command accepted; status not up yet. + admin_calls.append(admin_up) + if admin_up: + raise_issued.set() + + def gated_get_oper_status(): + operation = daemon.operation_runner.current + if (threading.current_thread().name == "bmcctld-op" and + operation is not None and + operation.item.action == bmcctld.ACTION_GRACEFUL_SHUTDOWN): + graceful_reads.append(1) + if len(graceful_reads) == 2: + # In the POWERING_OFF write, before the power call. + in_window.set() + assert operation.cancel.wait(2) + return real_get_oper_status() + + chassis.switch_host.set_admin_state = accepted_raise + chassis.switch_host.get_oper_status = gated_get_oper_status + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "CHASSIS_MODULE:ADMIN_UP", 5)) + daemon.operation_runner.process_next(timeout=0) + assert raise_issued.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "RACK_MGR_CMD:GRACEFUL_SHUT", 3)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.process_next(timeout=0) + assert in_window.wait(1) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "CRITICAL_SYSTEM_LEAK", 0)) + + daemon.operation_runner.process_next(timeout=0) + preempted = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert preempted[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_OFF + assert preempted[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_PREEMPTED + assert admin_calls == [True] + daemon.operation_runner.process_next(timeout=0) + successor = daemon.operation_runner.current + assert successor is not None + successor.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert admin_calls == [True, False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_TRIGGER] == "CRITICAL_SYSTEM_LEAK" + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + + def test_p_refused_retry_after_failed_raise_cannot_absorb_off_successor( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + admin_calls = [] + chassis.switch_host.set_admin_state = admin_calls.append + daemon.controller._verify_oper_status = MagicMock(side_effect=[ + RuntimeError("status read failed"), + bmcctld.PowerCallResult.CONFIRMED, + ]) + + # A raise is issued and then fails: the label stays transitional. + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "CHASSIS_MODULE:ADMIN_UP", 5)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + failed = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert failed[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_ON + assert failed[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_ON_FAILED + + # A refused retry restores the label it found and replaces the result. + daemon.critical_event_checker.has_any_critical_event = MagicMock( + return_value=True) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "RACK_MGR_CMD:POWER_ON", 5)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + refused = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert refused[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_ON + assert refused[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_OFF_LEAK_BLOCKED + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "CRITICAL_SYSTEM_LEAK", 0)) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert operation is not None + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert admin_calls == [True, False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + + def test_p_abandoned_issued_raise_cannot_absorb_off_successor(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + admin_calls = [] + chassis.switch_host.set_admin_state = admin_calls.append + # A previous daemon life died after issuing a raise. + daemon.controller.write_operation_start( + TEST_REQUEST_ID, "CHASSIS_MODULE:ADMIN_UP") + daemon.controller._update_host_state(bmcctld.SWITCH_HOST_POWERING_ON) + daemon.controller.init_host_state() + restarted = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert restarted[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_ABANDONED + assert restarted[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_ON + + # A retry refused by the leak gate replaces the result but keeps the label. + daemon.critical_event_checker.has_any_critical_event = MagicMock( + return_value=True) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "RACK_MGR_CMD:POWER_ON", 5)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + refused = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert refused[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_OFF_LEAK_BLOCKED + assert refused[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_ON + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "CRITICAL_SYSTEM_LEAK", 0)) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert operation is not None + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert admin_calls == [False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + + @pytest.mark.parametrize("raise_action,failure", [ + (bmcctld.ACTION_POWER_ON, "timeout"), + (bmcctld.ACTION_POWER_ON, "exception"), + (bmcctld.ACTION_POWER_CYCLE, "timeout"), + (bmcctld.ACTION_POWER_CYCLE, "exception"), + ]) + def test_p_unconfirmed_raise_survives_restart_and_cannot_absorb_off_successor( + self, chassis, monkeypatch, raise_action, failure): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + calls = [] + chassis.switch_host.set_admin_state = lambda up: calls.append(up) + chassis.switch_host.do_power_cycle = lambda: calls.append("cycle") + if failure == "timeout": + monkeypatch.setattr(bmcctld, "POWER_VERIFY_TIMEOUT_SECS", 0) + else: + daemon.controller._verify_oper_status = MagicMock(side_effect=[ + RuntimeError("status read failed"), + bmcctld.PowerCallResult.CONFIRMED, + ]) + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + raise_action, "raise", bmcctld.action_priority(raise_action))) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + transitional = (bmcctld.SWITCH_HOST_POWERING_ON + if raise_action == bmcctld.ACTION_POWER_ON + else bmcctld.SWITCH_HOST_POWER_CYCLING) + failed = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert failed[bmcctld.FIELD_DEVICE_POWER_STATE] == transitional + assert failed[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_ON_FAILED + + # Daemon restart: the live sample is OFFLINE, the label is kept. + daemon.controller.init_host_state() + restarted = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert restarted[bmcctld.FIELD_DEVICE_POWER_STATE] == transitional + assert restarted[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_ON_FAILED + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "CRITICAL_SYSTEM_LEAK", 0)) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert operation is not None + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert calls == [True if raise_action == bmcctld.ACTION_POWER_ON + else "cycle", False] + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_SUCCESS + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + + @pytest.mark.parametrize("raise_action,stable_label", [ + (bmcctld.ACTION_POWER_ON, bmcctld.POWER_STATE_ON), + (bmcctld.ACTION_POWER_CYCLE, bmcctld.POWER_STATE_CYCLE), + ]) + @pytest.mark.parametrize("successor,priority", [ + (bmcctld.ACTION_POWER_OFF, 0), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, 3), + ]) + def test_p_stable_label_survives_restart_and_cannot_absorb_off_successor( + self, chassis, raise_action, stable_label, successor, priority): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=False) + set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=set_admin_state) + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + raise_action, "raise", bmcctld.action_priority(raise_action))) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + confirmed = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert confirmed[bmcctld.FIELD_DEVICE_POWER_STATE] == stable_label + + # The host goes offline on its own; the daemon restarts. + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon.controller.init_host_state() + restarted = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert restarted[bmcctld.FIELD_DEVICE_POWER_STATE] == stable_label + assert restarted[bmcctld.FIELD_DEVICE_STATUS] == bmcctld.SWITCH_HOST_OFFLINE + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + successor, "successor", priority)) + daemon.operation_runner.process_next(timeout=0) + operation = daemon.operation_runner.current + assert operation is not None + operation.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + assert chassis.switch_host.set_admin_state.call_args_list[-1] == call(False) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_OP_RESULT] in ( + bmcctld.OP_RESULT_SUCCESS, bmcctld.OP_RESULT_SUCCESS_FORCED) + + def test_p_confirmed_off_absorbs_repeat(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.controller.init_host_state() + set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=set_admin_state) + + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "RACK_MGR_CMD:POWER_OFF", 2)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + callback = MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_SHUTDOWN, "RACK_MGR_CMD:GRACEFUL_SHUT", 3, + on_complete=callback, rack_cmd_key="CMD_2")) + daemon.operation_runner.process_next(timeout=0) + + chassis.switch_host.set_admin_state.assert_called_once_with(False) + callback.assert_called_once_with(True, "guard_skip") + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_SUCCESS_FORCED + assert state[bmcctld.FIELD_OP_REASON] == bmcctld.OP_REASON_ALREADY_OFF + + def test_q_platform_calls_use_worker_while_operation_is_live(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + observations = [] + get_oper_status = chassis.switch_host.get_oper_status + set_admin_state = chassis.switch_host.set_admin_state + + def in_flight(): + current = daemon.operation_runner.current + return (current is not None and current.thread is not None and + current.thread.is_alive()) + + def observed_status(): + observations.append( + (threading.current_thread().name, in_flight())) + return get_oper_status() + + def observed_admin_state(up): + observations.append( + (threading.current_thread().name, in_flight())) + return set_admin_state(up) + + chassis.switch_host.get_oper_status = observed_status + chassis.switch_host.set_admin_state = observed_admin_state + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "thread-check", 2)) + daemon.operation_runner.process_next(timeout=0) + daemon.operation_runner.current.thread.join(2) + daemon.operation_runner.process_next(timeout=0) + + live_callers = [name for name, live in observations if live] + assert live_callers + assert set(live_callers) == {"bmcctld-op"} + + def test_q_stop_cancels_and_joins_worker_for_five_seconds(self, chassis): + daemon = self._make_daemon(chassis) + callback = MagicMock() + operation = _make_operation( + bmcctld.ACTION_POWER_ON, callback=callback) + operation.thread = MagicMock() + operation.thread.is_alive.return_value = True + daemon.operation_runner.current = operation + daemon.controller.write_operation_result = MagicMock() + + daemon.operation_runner.stop() + + assert operation.cancel.is_set() + operation.thread.join.assert_called_once_with(timeout=5) + assert daemon.operation_runner.current is operation + callback.assert_not_called() + daemon.controller.write_operation_result.assert_not_called() + + def test_q_callback_failure_does_not_wedge_reap(self, chassis): + daemon = self._make_daemon(chassis) + first = MagicMock() + third = MagicMock() + def raising(_success, _detail): + raise RuntimeError("callback failed") + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "callbacks", 5, + on_complete=first, rack_cmd_key="CMD_1") + operation = bmcctld.Operation( + item, TEST_REQUEST_ID, bmcctld.STAGE_POWER_ON_CONFIRMED) + operation.outcome = (bmcctld.OP_RESULT_SUCCESS, "-", True) + operation.joined_callbacks = [ + (raising, "CMD_2"), (third, "CMD_3")] + operation.thread = MagicMock() + operation.thread.is_alive.return_value = False + daemon.operation_runner.current = operation + daemon.operation_runner.log_error = MagicMock() + assert daemon.operation_runner._reap_if_done() is True + assert daemon.operation_runner.current is None + first.assert_called_once_with(True, bmcctld.OP_RESULT_SUCCESS) + third.assert_called_once_with(True, bmcctld.OP_RESULT_SUCCESS) + daemon.operation_runner.log_error.assert_called_once_with( + "CALLBACK_FAILED request_id={} cmd_key=CMD_2".format( + TEST_REQUEST_ID)) + + @pytest.mark.parametrize( + "guarded, expected_request_id", + [(True, TEST_REQUEST_ID), (False, "-")], + ids=["guard", "refusal"], + ) + def test_q_callback_failure_logs_path_identity( + self, guarded, expected_request_id, chassis): + daemon = self._make_daemon(chassis) + daemon.operation_runner.log_error = MagicMock() + daemon.operation_runner._new_request_id = MagicMock( + return_value=TEST_REQUEST_ID) + + def raising(_success, _detail): + raise RuntimeError("callback failed") + + if guarded: + daemon.controller.init_host_state() + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "guard", 2, + on_complete=raising, rack_cmd_key="CMD_GUARD") + else: + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + operation = _make_operation(bmcctld.ACTION_POWER_OFF) + operation.thread = MagicMock() + operation.thread.is_alive.return_value = True + daemon.operation_runner.current = operation + item = bmcctld.ActionItem( + bmcctld.ACTION_POWER_ON, "refuse", 5, + on_complete=raising, rack_cmd_key="CMD_REFUSE") + daemon.operation_runner.enqueue(item) + + daemon.operation_runner.process_next(timeout=0) + + cmd_key = "CMD_GUARD" if guarded else "CMD_REFUSE" + daemon.operation_runner.log_error.assert_called_once_with( + "CALLBACK_FAILED request_id={} cmd_key={}".format( + expected_request_id, cmd_key)) + + def test_q_missing_worker_outcome_is_abandoned(self, chassis): + daemon = self._make_daemon(chassis) + operation = _make_operation(bmcctld.ACTION_POWER_ON) + operation.thread = MagicMock() + operation.thread.is_alive.return_value = False + daemon.operation_runner.current = operation + daemon.operation_runner.log_error = MagicMock() + daemon.operation_runner._reap_if_done() + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == bmcctld.OP_RESULT_ABANDONED + assert state[bmcctld.FIELD_OP_REASON] == bmcctld.OP_REASON_UNCLASSIFIED + daemon.operation_runner.log_error.assert_called_once() + + def test_guard_does_not_treat_missing_op_result_as_in_flight(self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.controller._update_host_state(bmcctld.SWITCH_HOST_POWERING_OFF) + item = bmcctld.ActionItem(bmcctld.ACTION_POWER_OFF, "legacy", 2) + assert daemon.operation_runner._guard_should_skip(item) is False + + +class TestWorkerExceptionRecovery: + + def _make_runner(self, chassis): + with patch('sonic_platform.platform.Platform') as platform: + platform.return_value.get_chassis.return_value = chassis + daemon = bmcctld.BmcctldDaemon(bmcctld.SYSLOG_IDENTIFIER) + return daemon, daemon.operation_runner + + @pytest.mark.parametrize("action", [ + bmcctld.ACTION_GRACEFUL_SHUTDOWN, bmcctld.ACTION_GRACEFUL_RESTART]) + @pytest.mark.parametrize("scenario", ["graceful", "rpc_failure", "deadline", "classifier_error"]) + @pytest.mark.parametrize("close_error", [False, True]) + def test_channel_close_preserves_worker_outcome(self, chassis, action, scenario, close_error): + daemon, runner = self._make_runner(chassis) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + chassis.switch_host.set_admin_state = MagicMock(wraps=chassis.switch_host.set_admin_state) + chassis.switch_host.do_power_cycle = MagicMock() + daemon.graceful_shutdown._is_graceful_qualified = MagicMock(return_value=True) + daemon.policy_reader.get_graceful_shutdown_timeout = MagicMock(return_value=1) + operation = _make_operation(action) + clock = FakeClock() + operation.cancel.wait = MagicMock(side_effect=lambda delay: clock.advance(delay) or False) + requester = MagicMock() + requester.poll_status.return_value = _report( + "done [bmc-req:{}]".format(operation.request_id), active=scenario == "deadline") + if scenario == "rpc_failure": + requester.poll_status.side_effect = bmcctld.GnoiRpcError("poll failed") + if close_error: + requester.close.side_effect = bmcctld.GnoiRpcError("close failed") + daemon.gnoi_requester_factory = MagicMock(return_value=requester) + + with patch('bmcctld.time.monotonic', side_effect=clock), \ + patch('bmcctld.classify_report', wraps=bmcctld.classify_report) as classify: + if scenario == "classifier_error": + classify.side_effect = RuntimeError("classification failed") + runner._run_worker(operation) + + restart = action == bmcctld.ACTION_GRACEFUL_RESTART + restart_failed = restart and scenario == "classifier_error" + reason = { + "graceful": "-", "rpc_failure": bmcctld.OP_REASON_RPC_FAILURE, + "deadline": bmcctld.OP_REASON_DEADLINE, + "classifier_error": bmcctld.OP_REASON_UNCLASSIFIED, + }[scenario] + result = (bmcctld.OP_RESULT_POWER_ON_FAILED if restart_failed else + bmcctld.OP_RESULT_SUCCESS_GRACEFUL if scenario == "graceful" else + bmcctld.OP_RESULT_SUCCESS_FORCED) + assert operation.outcome == (result, reason, not restart_failed) + assert chassis.switch_host.set_admin_state.call_args_list == ( + [call(False), call(True)] if restart and not restart_failed else [call(False)]) + chassis.switch_host.do_power_cycle.assert_not_called() + daemon.gnoi_requester_factory.assert_called_once() + requester.open.assert_called_once() + requester.send_halt.assert_called_once() + requester.poll_status.assert_called_once() + requester.close.assert_called_once() + assert not operation.cancel.is_set() + + @pytest.mark.parametrize( + "command, action, expected_outcome, expected_cmd_status, expected_state", + [ + (bmcctld.CMD_GRACEFUL_SHUT, + bmcctld.ACTION_GRACEFUL_SHUTDOWN, + (bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_NOT_QUALIFIED, True), + bmcctld.CMD_STATUS_DONE, + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN), + (bmcctld.CMD_GRACEFUL_RESTART, + bmcctld.ACTION_GRACEFUL_RESTART, + (bmcctld.OP_RESULT_POWER_ON_FAILED, + bmcctld.OP_REASON_NOT_QUALIFIED, False), + bmcctld.CMD_STATUS_FAILED, + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN), + (bmcctld.CMD_POWER_OFF, + bmcctld.ACTION_POWER_OFF, + (bmcctld.OP_RESULT_SUCCESS, "-", True), + bmcctld.CMD_STATUS_DONE, + bmcctld.POWER_STATE_ON), + ], + ) + def test_downward_bookkeeping_failure_does_not_block_power_removal( + self, command, action, expected_outcome, expected_cmd_status, + expected_state, chassis): + daemon, runner = self._make_runner(chassis) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon.controller._update_host_state( + bmcctld.POWER_STATE_ON, bmcctld.SWITCH_HOST_ONLINE) + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=False) + command_key = "bookkeeping-failure" + daemon.event_handler._handle_rack_mgr_command(command_key, { + bmcctld.FIELD_COMMAND: command, + bmcctld.FIELD_STATUS: bmcctld.CMD_STATUS_PENDING, + }) + + get_power_state = daemon.controller.get_db_power_state + get_device_status = daemon.controller.get_db_device_status + + def fail_worker_power_state_read(): + if threading.current_thread().name == "bmcctld-op": + raise RuntimeError("power-state read failed") + return get_power_state() + + def fail_worker_device_status_read(): + if threading.current_thread().name == "bmcctld-op": + raise RuntimeError("device-status read failed") + return get_device_status() + + update_host_state = daemon.controller._update_host_state + + def fail_downward_state_writes(power_state, device_status=None): + if power_state in ( + bmcctld.SWITCH_HOST_POWERING_OFF, + bmcctld.POWER_STATE_OFF): + raise RuntimeError("power-state write failed") + return update_host_state(power_state, device_status) + + daemon.controller.get_db_power_state = MagicMock( + side_effect=fail_worker_power_state_read) + daemon.controller.get_db_device_status = MagicMock( + side_effect=fail_worker_device_status_read) + daemon.controller._update_host_state = MagicMock( + side_effect=fail_downward_state_writes) + set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=set_admin_state) + + assert runner.process_next(timeout=0) is True + operation = runner.current + assert operation.item.action == action + operation.thread.join(2) + assert operation.thread.is_alive() is False + runner.process_next(timeout=0) + + assert chassis.switch_host.set_admin_state.call_args_list == [ + call(False)] + assert chassis.switch_host.get_oper_status() == \ + MockModule.MODULE_STATUS_OFFLINE + assert operation.stage == bmcctld.STAGE_POWER_OFF_CONFIRMED + assert operation.outcome == expected_outcome + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == expected_outcome[0] + assert state[bmcctld.FIELD_OP_REASON] == expected_outcome[1] + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == expected_state + command_row = dict(daemon.event_handler._thread_database.table( + "STATE_DB", bmcctld.RACK_MANAGER_COMMAND_TABLE).get( + command_key)[1]) + assert command_row[bmcctld.FIELD_STATUS] == expected_cmd_status + assert command_row[bmcctld.FIELD_RESULT] == ( + "SUCCESS" if expected_outcome[2] + else bmcctld.OP_RESULT_POWER_ON_FAILED) + assert command_row[bmcctld.FIELD_REQUEST_ID] == operation.request_id + + def test_graceful_restart_bookkeeping_failure_preserves_leg_reason( + self, chassis): + daemon, runner = self._make_runner(chassis) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon.controller._update_host_state( + bmcctld.POWER_STATE_ON, bmcctld.SWITCH_HOST_ONLINE) + daemon.policy_reader.get_graceful_shutdown_timeout = MagicMock( + return_value=10) + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=True) + requester = MagicMock() + requester.poll_status.return_value = _report( + "done [bmc-req:{}]".format(TEST_REQUEST_ID)) + daemon.gnoi_requester_factory = MagicMock(return_value=requester) + + update_host_state = daemon.controller._update_host_state + + def fail_downward_state_writes(power_state, device_status=None): + if power_state in ( + bmcctld.SWITCH_HOST_POWERING_OFF, + bmcctld.POWER_STATE_OFF): + raise RuntimeError("power-state write failed") + return update_host_state(power_state, device_status) + + daemon.controller._update_host_state = MagicMock( + side_effect=fail_downward_state_writes) + set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=set_admin_state) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_RESTART) + + runner._run_worker(operation) + + assert operation.leg_graceful is True + assert operation.leg_reason == "-" + assert chassis.switch_host.set_admin_state.call_args_list == [ + call(False)] + assert operation.stage == bmcctld.STAGE_POWER_OFF_CONFIRMED + assert operation.outcome == ( + bmcctld.OP_RESULT_POWER_ON_FAILED, "-", False) + + @pytest.mark.parametrize( + "action, expected_reason", + [ + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, + bmcctld.OP_REASON_NOT_QUALIFIED), + (bmcctld.ACTION_GRACEFUL_RESTART, + bmcctld.OP_REASON_NOT_QUALIFIED), + (bmcctld.ACTION_POWER_OFF, "-"), + ], + ) + def test_downward_platform_exception_is_not_retried( + self, action, expected_reason, chassis): + daemon, runner = self._make_runner(chassis) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon.controller._update_host_state( + bmcctld.POWER_STATE_ON, bmcctld.SWITCH_HOST_ONLINE) + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=False) + chassis.switch_host.set_admin_state = MagicMock( + side_effect=RuntimeError("platform call failed")) + operation = _make_operation(action) + + runner._run_worker(operation) + + chassis.switch_host.set_admin_state.assert_called_once_with(False) + assert operation.stage == bmcctld.STAGE_POWER_OFF_ISSUED + assert operation.outcome == ( + bmcctld.OP_RESULT_POWER_OFF_FAILED, expected_reason, False) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_OFF + + @pytest.mark.parametrize( + "failure_point", + ["prior-state-read", "transitional-write", "critical-state-read"], + ) + def test_power_on_bookkeeping_failure_remains_fail_closed( + self, failure_point, chassis): + daemon, runner = self._make_runner(chassis) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + daemon.controller._update_host_state( + bmcctld.POWER_STATE_OFF, bmcctld.SWITCH_HOST_OFFLINE) + chassis.switch_host.set_admin_state = MagicMock() + + if failure_point == "prior-state-read": + daemon.controller.get_db_power_state = MagicMock( + side_effect=RuntimeError("power-state read failed")) + elif failure_point == "transitional-write": + update_host_state = daemon.controller._update_host_state + + def fail_powering_on_write(power_state, device_status=None): + if power_state == bmcctld.SWITCH_HOST_POWERING_ON: + raise RuntimeError("power-state write failed") + return update_host_state(power_state, device_status) + + daemon.controller._update_host_state = MagicMock( + side_effect=fail_powering_on_write) + else: + daemon.controller.critical_event_checker.has_any_critical_event = \ + MagicMock(side_effect=RuntimeError( + "critical-state read failed")) + + operation = _make_operation(bmcctld.ACTION_POWER_ON) + runner._run_worker(operation) + + chassis.switch_host.set_admin_state.assert_not_called() + assert operation.stage == bmcctld.STAGE_POWER_ON_PENDING + assert operation.outcome == ( + bmcctld.OP_RESULT_POWER_ON_FAILED, "-", False) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.POWER_STATE_OFF + assert state[bmcctld.FIELD_DEVICE_STATUS] == \ + bmcctld.SWITCH_HOST_OFFLINE + if failure_point == "transitional-write": + assert daemon.controller._update_host_state.call_args_list == [ + call(bmcctld.SWITCH_HOST_POWERING_ON), + call(bmcctld.POWER_STATE_OFF, bmcctld.SWITCH_HOST_OFFLINE), + ] + + @pytest.mark.parametrize( + "command, action, expected_reason, expected_state", + [ + (bmcctld.CMD_GRACEFUL_SHUT, + bmcctld.ACTION_GRACEFUL_SHUTDOWN, + bmcctld.OP_REASON_NOT_QUALIFIED, + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN), + (bmcctld.CMD_GRACEFUL_RESTART, + bmcctld.ACTION_GRACEFUL_RESTART, + bmcctld.OP_REASON_NOT_QUALIFIED, + bmcctld.SWITCH_HOST_GRACEFUL_SHUTTING_DOWN), + (bmcctld.CMD_POWER_OFF, + bmcctld.ACTION_POWER_OFF, "-", bmcctld.POWER_STATE_ON), + ], + ) + def test_bookkeeping_and_platform_failures_issue_one_downward_call( + self, command, action, expected_reason, expected_state, chassis): + daemon, runner = self._make_runner(chassis) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon.controller._update_host_state( + bmcctld.POWER_STATE_ON, bmcctld.SWITCH_HOST_ONLINE) + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=False) + command_key = "bookkeeping-and-platform-failure" + daemon.event_handler._handle_rack_mgr_command(command_key, { + bmcctld.FIELD_COMMAND: command, + bmcctld.FIELD_STATUS: bmcctld.CMD_STATUS_PENDING, + }) + + get_power_state = daemon.controller.get_db_power_state + + def fail_worker_power_state_read(): + if threading.current_thread().name == "bmcctld-op": + raise RuntimeError("power-state read failed") + return get_power_state() + + update_host_state = daemon.controller._update_host_state + + def fail_downward_state_writes(power_state, device_status=None): + if power_state in ( + bmcctld.SWITCH_HOST_POWERING_OFF, + bmcctld.POWER_STATE_OFF): + raise RuntimeError("power-state write failed") + return update_host_state(power_state, device_status) + + daemon.controller.get_db_power_state = MagicMock( + side_effect=fail_worker_power_state_read) + daemon.controller.get_db_device_status = MagicMock( + side_effect=RuntimeError("device-status read failed")) + daemon.controller._update_host_state = MagicMock( + side_effect=fail_downward_state_writes) + chassis.switch_host.set_admin_state = MagicMock( + side_effect=RuntimeError("platform call failed")) + + assert runner.process_next(timeout=0) is True + operation = runner.current + assert operation.item.action == action + operation.thread.join(2) + assert operation.thread.is_alive() is False + runner.process_next(timeout=0) + + chassis.switch_host.set_admin_state.assert_called_once_with(False) + assert operation.stage == bmcctld.STAGE_POWER_OFF_ISSUED + assert operation.outcome == ( + bmcctld.OP_RESULT_POWER_OFF_FAILED, expected_reason, False) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_OFF_FAILED + assert state[bmcctld.FIELD_OP_REASON] == expected_reason + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == expected_state + command_row = dict(daemon.event_handler._thread_database.table( + "STATE_DB", bmcctld.RACK_MANAGER_COMMAND_TABLE).get( + command_key)[1]) + assert command_row[bmcctld.FIELD_STATUS] == \ + bmcctld.CMD_STATUS_FAILED + assert command_row[bmcctld.FIELD_RESULT] == \ + bmcctld.OP_RESULT_POWER_OFF_FAILED + assert command_row[bmcctld.FIELD_REQUEST_ID] == operation.request_id + + @pytest.mark.parametrize( + "action, stage, power_result, expected, expected_power_calls", + [ + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, bmcctld.STAGE_HANDSHAKE, + bmcctld.PowerCallResult.CONFIRMED, + (bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_UNCLASSIFIED, True), 1), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, bmcctld.STAGE_HANDSHAKE, + bmcctld.PowerCallResult.NOT_CONFIRMED, + (bmcctld.OP_RESULT_POWER_OFF_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 1), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, bmcctld.STAGE_HANDSHAKE, + bmcctld.PowerCallResult.CANCELLED, + (bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, False), 1), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, + bmcctld.STAGE_POWER_OFF_ISSUED, + bmcctld.PowerCallResult.CONFIRMED, + (bmcctld.OP_RESULT_POWER_OFF_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 0), + (bmcctld.ACTION_POWER_OFF, bmcctld.STAGE_POWER_OFF_PENDING, + bmcctld.PowerCallResult.CONFIRMED, + (bmcctld.OP_RESULT_SUCCESS, "-", True), 1), + (bmcctld.ACTION_POWER_OFF, bmcctld.STAGE_POWER_OFF_PENDING, + bmcctld.PowerCallResult.NOT_CONFIRMED, + (bmcctld.OP_RESULT_POWER_OFF_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 1), + (bmcctld.ACTION_POWER_OFF, bmcctld.STAGE_POWER_OFF_PENDING, + bmcctld.PowerCallResult.CANCELLED, + (bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, False), 1), + (bmcctld.ACTION_POWER_OFF, bmcctld.STAGE_POWER_OFF_ISSUED, + bmcctld.PowerCallResult.CONFIRMED, + (bmcctld.OP_RESULT_POWER_OFF_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 0), + (bmcctld.ACTION_POWER_CYCLE, bmcctld.STAGE_POWER_CYCLE_PENDING, + bmcctld.PowerCallResult.CONFIRMED, + (bmcctld.OP_RESULT_POWER_ON_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 0), + (bmcctld.ACTION_POWER_CYCLE, bmcctld.STAGE_POWER_CYCLE_ISSUED, + bmcctld.PowerCallResult.CONFIRMED, + (bmcctld.OP_RESULT_POWER_ON_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 0), + (bmcctld.ACTION_POWER_ON, bmcctld.STAGE_POWER_ON_PENDING, + bmcctld.PowerCallResult.CONFIRMED, + (bmcctld.OP_RESULT_POWER_ON_FAILED, "-", False), 0), + (bmcctld.ACTION_POWER_ON, bmcctld.STAGE_POWER_ON_ISSUED, + bmcctld.PowerCallResult.CONFIRMED, + (bmcctld.OP_RESULT_POWER_ON_FAILED, "-", False), 0), + ], + ) + def test_q_stage_recovery_never_reissues_ambiguous_calls( + self, action, stage, power_result, expected, + expected_power_calls, chassis): + daemon, runner = self._make_runner(chassis) + operation = _make_operation(action) + operation.stage = stage + daemon.controller.power_off = MagicMock(return_value=power_result) + daemon.controller.power_on = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + daemon.controller.power_cycle = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) + assert runner._recover_worker_exception(operation) == expected + assert daemon.controller.power_off.call_count == expected_power_calls + daemon.controller.power_on.assert_not_called() + daemon.controller.power_cycle.assert_not_called() + + @pytest.mark.parametrize( + "action, stage, leg_graceful, leg_reason, expected", + [ + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, + bmcctld.STAGE_POWER_OFF_CONFIRMED, True, None, + (bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True)), + (bmcctld.ACTION_GRACEFUL_SHUTDOWN, + bmcctld.STAGE_POWER_OFF_CONFIRMED, False, + bmcctld.OP_REASON_DEADLINE, + (bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_DEADLINE, True)), + (bmcctld.ACTION_GRACEFUL_RESTART, + bmcctld.STAGE_POWER_ON_CONFIRMED, True, None, + (bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True)), + (bmcctld.ACTION_GRACEFUL_RESTART, + bmcctld.STAGE_POWER_ON_CONFIRMED, False, + bmcctld.OP_REASON_CHECK_FAILED, + (bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_CHECK_FAILED, True)), + (bmcctld.ACTION_POWER_OFF, + bmcctld.STAGE_POWER_OFF_CONFIRMED, False, None, + (bmcctld.OP_RESULT_SUCCESS, "-", True)), + (bmcctld.ACTION_POWER_ON, + bmcctld.STAGE_POWER_ON_CONFIRMED, False, None, + (bmcctld.OP_RESULT_SUCCESS, "-", True)), + (bmcctld.ACTION_POWER_CYCLE, + bmcctld.STAGE_POWER_CYCLE_CONFIRMED, False, None, + (bmcctld.OP_RESULT_SUCCESS, "-", True)), + ], + ) + def test_q_confirmed_stage_never_becomes_false_failure( + self, action, stage, leg_graceful, leg_reason, expected, + chassis): + daemon, runner = self._make_runner(chassis) + operation = _make_operation(action) + operation.stage = stage + operation.leg_graceful = leg_graceful + operation.leg_reason = leg_reason + daemon.controller.power_off = MagicMock() + daemon.controller.power_on = MagicMock() + daemon.controller.power_cycle = MagicMock() + assert runner._recover_worker_exception(operation) == expected + daemon.controller.power_off.assert_not_called() + daemon.controller.power_on.assert_not_called() + daemon.controller.power_cycle.assert_not_called() + + @pytest.mark.parametrize( + "stage, power_result, leg_reason, expected, expected_power_calls", + [ + (bmcctld.STAGE_HANDSHAKE, + bmcctld.PowerCallResult.CONFIRMED, None, + (bmcctld.OP_RESULT_POWER_ON_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 1), + (bmcctld.STAGE_HANDSHAKE, + bmcctld.PowerCallResult.NOT_CONFIRMED, None, + (bmcctld.OP_RESULT_POWER_OFF_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 1), + (bmcctld.STAGE_POWER_OFF_ISSUED, + bmcctld.PowerCallResult.CONFIRMED, None, + (bmcctld.OP_RESULT_POWER_OFF_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 0), + (bmcctld.STAGE_POWER_OFF_CONFIRMED, + bmcctld.PowerCallResult.CONFIRMED, None, + (bmcctld.OP_RESULT_POWER_ON_FAILED, + bmcctld.OP_REASON_UNCLASSIFIED, False), 0), + (bmcctld.STAGE_PAUSE, + bmcctld.PowerCallResult.CONFIRMED, + bmcctld.OP_REASON_DEADLINE, + (bmcctld.OP_RESULT_POWER_ON_FAILED, + bmcctld.OP_REASON_DEADLINE, False), 0), + (bmcctld.STAGE_POWER_ON_ISSUED, + bmcctld.PowerCallResult.CONFIRMED, + bmcctld.OP_REASON_CHECK_FAILED, + (bmcctld.OP_RESULT_POWER_ON_FAILED, + bmcctld.OP_REASON_CHECK_FAILED, False), 0), + ], + ) + def test_q_restart_stage_recovery_never_adds_a_raise( + self, stage, power_result, leg_reason, expected, + expected_power_calls, chassis): + daemon, runner = self._make_runner(chassis) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_RESTART) + operation.stage = stage + operation.leg_reason = leg_reason + daemon.controller.power_off = MagicMock(return_value=power_result) + daemon.controller.power_on = MagicMock() + daemon.controller.power_cycle = MagicMock() + + assert runner._recover_worker_exception(operation) == expected + assert daemon.controller.power_off.call_count == expected_power_calls + daemon.controller.power_on.assert_not_called() + daemon.controller.power_cycle.assert_not_called() + + @pytest.mark.parametrize( + "action, initial_status, final_state, confirmed_stage", + [ + (bmcctld.ACTION_POWER_OFF, MockModule.MODULE_STATUS_ONLINE, + bmcctld.POWER_STATE_OFF, bmcctld.STAGE_POWER_OFF_CONFIRMED), + (bmcctld.ACTION_POWER_ON, MockModule.MODULE_STATUS_OFFLINE, + bmcctld.POWER_STATE_ON, bmcctld.STAGE_POWER_ON_CONFIRMED), + (bmcctld.ACTION_POWER_CYCLE, MockModule.MODULE_STATUS_ONLINE, + bmcctld.POWER_STATE_CYCLE, + bmcctld.STAGE_POWER_CYCLE_CONFIRMED), + ], + ) + def test_q_confirmed_wrapper_bookkeeping_failure_reaches_recovery( + self, action, initial_status, final_state, confirmed_stage, + chassis): + daemon, runner = self._make_runner(chassis) + chassis.switch_host.set_oper_status(initial_status) + operation = _make_operation(action) + update_state = daemon.controller._update_host_state + + def fail_final_state(power_state, device_status=None): + if power_state == final_state: + raise RuntimeError("final state write failed") + return update_state(power_state, device_status) + + daemon.controller._update_host_state = MagicMock( + side_effect=fail_final_state) + runner._run_worker(operation) + + assert operation.stage == confirmed_stage + assert operation.outcome == ( + bmcctld.OP_RESULT_SUCCESS, "-", True) + + def test_graceful_shutdown_transitional_write_failure_preserves_reason( + self, chassis): + daemon, runner = self._make_runner(chassis) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN) + daemon.graceful_shutdown._is_graceful_qualified = MagicMock( + return_value=False) + + update_state = daemon.controller._update_host_state + failed_once = [False] + + def fail_first_powering_off(power_state, device_status=None): + if power_state == bmcctld.SWITCH_HOST_POWERING_OFF and \ + not failed_once[0]: + failed_once[0] = True + raise RuntimeError("transitional state write failed") + return update_state(power_state, device_status) + + daemon.controller._update_host_state = MagicMock( + side_effect=fail_first_powering_off) + set_admin_state = chassis.switch_host.set_admin_state + chassis.switch_host.set_admin_state = MagicMock( + side_effect=set_admin_state) + + runner._run_worker(operation) + + assert failed_once[0] is True + chassis.switch_host.set_admin_state.assert_called_once_with(False) + assert operation.stage == bmcctld.STAGE_POWER_OFF_CONFIRMED + assert operation.outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_NOT_QUALIFIED, + True, + ) + + def test_q_recovery_power_off_always_leaves_an_outcome(self, chassis): + daemon, runner = self._make_runner(chassis) + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN) + daemon.graceful_shutdown.execute = MagicMock( + side_effect=RuntimeError("operation body failed")) + update_state = daemon.controller._update_host_state + + def fail_final_state(power_state, device_status=None): + if power_state == bmcctld.POWER_STATE_OFF: + raise RuntimeError("final state write failed") + return update_state(power_state, device_status) + + daemon.controller._update_host_state = MagicMock( + side_effect=fail_final_state) + runner._run_worker(operation) + + assert chassis.switch_host.get_admin_state() is False + assert operation.stage == bmcctld.STAGE_POWER_OFF_CONFIRMED + assert operation.outcome == ( + bmcctld.OP_RESULT_SUCCESS_FORCED, + bmcctld.OP_REASON_UNCLASSIFIED, + True, + ) + + def test_q_cancel_during_recovery_fallback_prevents_retry(self, chassis): + daemon, runner = self._make_runner(chassis) + operation = _make_operation(bmcctld.ACTION_POWER_OFF) + + def cancel_then_raise(_operation, cancel): + cancel.set() + raise RuntimeError("fallback call failed after cancellation") + + daemon.controller.power_off = MagicMock(side_effect=cancel_then_raise) + assert runner._recover_worker_exception(operation) == ( + bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, + False, + ) + daemon.controller.power_off.assert_called_once_with( + operation, operation.cancel) + + def test_q_cancel_wins_before_exception_recovery_call(self, chassis): + daemon, runner = self._make_runner(chassis) + operation = _make_operation(bmcctld.ACTION_GRACEFUL_SHUTDOWN) + operation.cancel.set() + daemon.controller.power_off = MagicMock() + assert runner._recover_worker_exception(operation) == ( + bmcctld.OP_RESULT_PREEMPTED, + bmcctld.OP_REASON_PREEMPTED, + False, + ) + daemon.controller.power_off.assert_not_called() # -------------------------------------------------------------------------- @@ -1193,6 +5579,7 @@ def test_skips_power_on_when_admin_status_down(self, chassis): daemon.policy_reader.get_switch_host_admin_status = MagicMock(return_value=bmcctld.ADMIN_DOWN) daemon.controller.power_on = MagicMock() daemon._initial_power_on_sequence() + assert daemon.action_queue.empty() daemon.controller.power_on.assert_not_called() def test_skips_power_on_when_admin_status_not_set(self, chassis): @@ -1203,15 +5590,18 @@ def test_skips_power_on_when_admin_status_not_set(self, chassis): daemon.policy_reader.get_switch_host_admin_status = MagicMock(return_value=bmcctld.ADMIN_DOWN) daemon.controller.power_on = MagicMock() daemon._initial_power_on_sequence() + assert daemon.action_queue.empty() daemon.controller.power_on.assert_not_called() def test_powers_on_when_no_leak_and_host_offline(self, chassis): chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) daemon = self._make_daemon(chassis) daemon.critical_event_checker.has_any_critical_event = MagicMock(return_value=False) - daemon.controller.power_on = MagicMock(return_value=True) daemon._initial_power_on_sequence() - daemon.controller.power_on.assert_called_once() + item = _dequeue_item(daemon.action_queue) + assert item.action == bmcctld.ACTION_POWER_ON + assert item.event_desc == "STARTUP" + assert item.startup is True def test_skips_power_on_if_critical_leak_present(self, chassis): chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) @@ -1219,6 +5609,7 @@ def test_skips_power_on_if_critical_leak_present(self, chassis): daemon.critical_event_checker.has_any_critical_event = MagicMock(return_value=True) daemon.controller.power_on = MagicMock() daemon._initial_power_on_sequence() + assert daemon.action_queue.empty() daemon.controller.power_on.assert_not_called() def test_refreshes_state_when_already_online(self, chassis): @@ -1228,6 +5619,7 @@ def test_refreshes_state_when_already_online(self, chassis): daemon.controller.power_on = MagicMock() daemon.controller.init_host_state = MagicMock() daemon._initial_power_on_sequence() + assert daemon.action_queue.empty() daemon.controller.power_on.assert_not_called() daemon.controller.init_host_state.assert_called_once() @@ -1237,7 +5629,6 @@ def test_boot_delay_skipped_when_system_uptime_exceeds_delay(self, chassis): daemon = self._make_daemon(chassis) daemon.policy_reader.get_power_on_delay = MagicMock(return_value=60) daemon.critical_event_checker.has_any_critical_event = MagicMock(return_value=False) - daemon.controller.power_on = MagicMock(return_value=True) # Pretend the system has been up for 10 minutes — bmcctld restart mid-life # must not re-arm the full 60s delay. with patch('time.clock_gettime', return_value=600): @@ -1245,7 +5636,8 @@ def test_boot_delay_skipped_when_system_uptime_exceeds_delay(self, chassis): daemon._initial_power_on_sequence() elapsed = time.monotonic() - t0 assert elapsed < 1.0, "boot delay should have been skipped (elapsed={:.2f}s)".format(elapsed) - daemon.controller.power_on.assert_called_once() + item = _dequeue_item(daemon.action_queue) + assert item.action == bmcctld.ACTION_POWER_ON def test_stop_event_during_boot_delay_skips_sequence(self, chassis): daemon = self._make_daemon(chassis) @@ -1255,6 +5647,7 @@ def test_stop_event_during_boot_delay_skips_sequence(self, chassis): daemon.stop_event.set() # Signal stop before delay expires with patch('time.clock_gettime', return_value=0): daemon._initial_power_on_sequence() + assert daemon.action_queue.empty() daemon.controller.power_on.assert_not_called() def test_boot_delay_processes_queued_actions(self, chassis): @@ -1264,10 +5657,12 @@ def test_boot_delay_processes_queued_actions(self, chassis): # Use a small non-zero delay so the queue-drain loop runs at least once daemon.policy_reader.get_power_on_delay = MagicMock(return_value=1) daemon.critical_event_checker.has_any_critical_event = MagicMock(return_value=False) - daemon.controller.power_off = MagicMock(return_value=True) + daemon.controller.power_off = MagicMock( + return_value=bmcctld.PowerCallResult.CONFIRMED) # Simulate a POWER_OFF arriving from Rack Manager during boot delay chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) - daemon.action_queue.put(bmcctld.ActionItem(bmcctld.ACTION_POWER_OFF, "RACK_MGR_BOOT_DELAY")) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "RACK_MGR_BOOT_DELAY", 2)) # Force system uptime to 0 so the full configured delay applies. with patch('time.clock_gettime', return_value=0): daemon._initial_power_on_sequence() @@ -1275,6 +5670,102 @@ def test_boot_delay_processes_queued_actions(self, chassis): assert daemon.action_queue.empty() daemon.controller.power_off.assert_called_once() + def test_q_boot_delay_hands_off_live_worker_without_startup_tail( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.policy_reader.get_power_on_delay = MagicMock(return_value=1) + daemon._rack_mgr_power_cmd_executed = MagicMock(return_value=False) + daemon.critical_event_checker.has_any_critical_event = MagicMock( + return_value=False) + daemon.controller.init_host_state = MagicMock() + verifying = threading.Event() + release = threading.Event() + + def unconfirmed_power_off(up): + assert up is False + chassis.switch_host._admin_state = up + + def blocked_verify(_expected, _timeout, _context, cancel=None): + verifying.set() + assert release.wait(3) + return bmcctld.PowerCallResult.NOT_CONFIRMED + + chassis.switch_host.set_admin_state = MagicMock( + side_effect=unconfirmed_power_off) + daemon.controller._verify_oper_status = MagicMock( + side_effect=blocked_verify) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "boot-delay-worker", 2)) + + with patch('time.clock_gettime', return_value=0): + daemon._initial_power_on_sequence() + + assert verifying.is_set() + assert daemon.operation_runner.current is not None + assert daemon.operation_runner.current.thread.is_alive() + assert daemon.action_queue.empty() + daemon._rack_mgr_power_cmd_executed.assert_not_called() + daemon.critical_event_checker.has_any_critical_event.assert_not_called() + daemon.controller.init_host_state.assert_not_called() + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_OFF + assert state[bmcctld.FIELD_OP_RESULT] == "-" + + release.set() + daemon.operation_runner.current.thread.join(1) + daemon.operation_runner.process_next(timeout=0) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_OFF + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_OFF_FAILED + + def test_q_boot_delay_hands_off_reaped_worker_without_startup_tail( + self, chassis): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + daemon.policy_reader.get_power_on_delay = MagicMock(return_value=2) + daemon._rack_mgr_power_cmd_executed = MagicMock(return_value=False) + daemon.critical_event_checker.has_any_critical_event = MagicMock( + return_value=False) + daemon.controller.init_host_state = MagicMock() + + def unconfirmed_power_off(up): + assert up is False + chassis.switch_host._admin_state = up + + chassis.switch_host.set_admin_state = MagicMock( + side_effect=unconfirmed_power_off) + daemon.controller._verify_oper_status = MagicMock( + return_value=bmcctld.PowerCallResult.NOT_CONFIRMED) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "boot-delay-reaped-worker", 2)) + + with patch('time.clock_gettime', return_value=0): + daemon._initial_power_on_sequence() + + assert daemon.operation_runner.current is None + assert daemon.action_queue.empty() + daemon._rack_mgr_power_cmd_executed.assert_not_called() + daemon.critical_event_checker.has_any_critical_event.assert_not_called() + daemon.controller.init_host_state.assert_not_called() + chassis.switch_host.set_admin_state.assert_called_once_with(False) + state = dict(daemon.controller.host_state_table.get( + bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_DEVICE_POWER_STATE] == \ + bmcctld.SWITCH_HOST_POWERING_OFF + assert state[bmcctld.FIELD_OP_RESULT] == \ + bmcctld.OP_RESULT_POWER_OFF_FAILED + + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + retry = bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "retry-after-unconfirmed-off", 2) + assert daemon.operation_runner._guard_should_skip(retry) is False + def test_rack_mgr_power_off_during_boot_delay_skips_auto_power_on(self, chassis): """If Rack Manager POWER_OFF cmd executed during boot delay, automatic power-on is skipped.""" chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) @@ -1283,6 +5774,7 @@ def test_rack_mgr_power_off_during_boot_delay_skips_auto_power_on(self, chassis) daemon.controller.power_on = MagicMock(return_value=True) daemon._rack_mgr_power_cmd_executed = MagicMock(return_value=True) daemon._initial_power_on_sequence() + assert daemon.action_queue.empty() daemon.controller.power_on.assert_not_called() def test_rack_mgr_power_on_during_boot_delay_skips_auto_power_on(self, chassis): @@ -1293,12 +5785,14 @@ def test_rack_mgr_power_on_during_boot_delay_skips_auto_power_on(self, chassis): daemon.controller.power_on = MagicMock(return_value=True) daemon._rack_mgr_power_cmd_executed = MagicMock(return_value=True) daemon._initial_power_on_sequence() + assert daemon.action_queue.empty() daemon.controller.power_on.assert_not_called() def test_rack_mgr_power_cmd_executed_detects_done_power_off(self, chassis): """_rack_mgr_power_cmd_executed returns True when POWER_OFF is DONE in RACK_MANAGER_COMMAND.""" daemon = self._make_daemon(chassis) - tbl = Table(daemon.event_handler.state_db, bmcctld.RACK_MANAGER_COMMAND_TABLE) + tbl = Table(daemon._thread_database.connection("STATE_DB"), + bmcctld.RACK_MANAGER_COMMAND_TABLE) tbl.set("CMD_1", FieldValuePairs([ (bmcctld.FIELD_COMMAND, bmcctld.CMD_POWER_OFF), (bmcctld.FIELD_STATUS, bmcctld.CMD_STATUS_DONE), @@ -1309,7 +5803,8 @@ def test_rack_mgr_power_cmd_executed_detects_done_power_off(self, chassis): def test_rack_mgr_power_cmd_executed_detects_in_progress_power_on(self, chassis): """_rack_mgr_power_cmd_executed returns True when POWER_ON is IN_PROGRESS.""" daemon = self._make_daemon(chassis) - tbl = Table(daemon.event_handler.state_db, bmcctld.RACK_MANAGER_COMMAND_TABLE) + tbl = Table(daemon._thread_database.connection("STATE_DB"), + bmcctld.RACK_MANAGER_COMMAND_TABLE) tbl.set("CMD_1", FieldValuePairs([ (bmcctld.FIELD_COMMAND, bmcctld.CMD_POWER_ON), (bmcctld.FIELD_STATUS, bmcctld.CMD_STATUS_IN_PROGRESS), @@ -1318,9 +5813,9 @@ def test_rack_mgr_power_cmd_executed_detects_in_progress_power_on(self, chassis) assert daemon._rack_mgr_power_cmd_executed() is True def test_rack_mgr_power_cmd_executed_ignores_power_cycle(self, chassis): - """_rack_mgr_power_cmd_executed returns False for POWER_CYCLE (not POWER_ON/OFF/GRACEFUL_SHUT).""" daemon = self._make_daemon(chassis) - tbl = Table(daemon.event_handler.state_db, bmcctld.RACK_MANAGER_COMMAND_TABLE) + tbl = Table(daemon._thread_database.connection("STATE_DB"), + bmcctld.RACK_MANAGER_COMMAND_TABLE) tbl.set("CMD_1", FieldValuePairs([ (bmcctld.FIELD_COMMAND, bmcctld.CMD_POWER_CYCLE), (bmcctld.FIELD_STATUS, bmcctld.CMD_STATUS_DONE), @@ -1331,7 +5826,8 @@ def test_rack_mgr_power_cmd_executed_ignores_power_cycle(self, chassis): def test_rack_mgr_power_cmd_executed_detects_graceful_shut(self, chassis): """_rack_mgr_power_cmd_executed returns True when GRACEFUL_SHUT is DONE.""" daemon = self._make_daemon(chassis) - tbl = Table(daemon.event_handler.state_db, bmcctld.RACK_MANAGER_COMMAND_TABLE) + tbl = Table(daemon._thread_database.connection("STATE_DB"), + bmcctld.RACK_MANAGER_COMMAND_TABLE) tbl.set("CMD_1", FieldValuePairs([ (bmcctld.FIELD_COMMAND, bmcctld.CMD_GRACEFUL_SHUT), (bmcctld.FIELD_STATUS, bmcctld.CMD_STATUS_DONE), @@ -1339,10 +5835,27 @@ def test_rack_mgr_power_cmd_executed_detects_graceful_shut(self, chassis): with patch('bmcctld.swsscommon.Table', return_value=tbl): assert daemon._rack_mgr_power_cmd_executed() is True + @pytest.mark.parametrize("status", [ + bmcctld.CMD_STATUS_IN_PROGRESS, + bmcctld.CMD_STATUS_DONE, + ]) + def test_rack_mgr_power_cmd_executed_detects_graceful_restart( + self, chassis, status): + daemon = self._make_daemon(chassis) + tbl = Table(daemon._thread_database.connection("STATE_DB"), + bmcctld.RACK_MANAGER_COMMAND_TABLE) + tbl.set("CMD_1", FieldValuePairs([ + (bmcctld.FIELD_COMMAND, bmcctld.CMD_GRACEFUL_RESTART), + (bmcctld.FIELD_STATUS, status), + ])) + with patch('bmcctld.swsscommon.Table', return_value=tbl): + assert daemon._rack_mgr_power_cmd_executed() is True + def test_rack_mgr_power_cmd_executed_ignores_pending(self, chassis): """_rack_mgr_power_cmd_executed returns False when command is still PENDING.""" daemon = self._make_daemon(chassis) - tbl = Table(daemon.event_handler.state_db, bmcctld.RACK_MANAGER_COMMAND_TABLE) + tbl = Table(daemon._thread_database.connection("STATE_DB"), + bmcctld.RACK_MANAGER_COMMAND_TABLE) tbl.set("CMD_1", FieldValuePairs([ (bmcctld.FIELD_COMMAND, bmcctld.CMD_POWER_OFF), (bmcctld.FIELD_STATUS, bmcctld.CMD_STATUS_PENDING), @@ -1353,7 +5866,8 @@ def test_rack_mgr_power_cmd_executed_ignores_pending(self, chassis): def test_rack_mgr_power_cmd_executed_empty_table(self, chassis): """_rack_mgr_power_cmd_executed returns False when no commands exist.""" daemon = self._make_daemon(chassis) - tbl = Table(daemon.event_handler.state_db, bmcctld.RACK_MANAGER_COMMAND_TABLE) + tbl = Table(daemon._thread_database.connection("STATE_DB"), + bmcctld.RACK_MANAGER_COMMAND_TABLE) with patch('bmcctld.swsscommon.Table', return_value=tbl): assert daemon._rack_mgr_power_cmd_executed() is False @@ -1381,8 +5895,8 @@ def test_warm_boot_skips_power_on_delay(self, chassis): daemon._initial_power_on_sequence() # Boot delay is skipped → get_power_on_delay must NOT be called daemon.policy_reader.get_power_on_delay.assert_not_called() - # And we still proceeded to the power-on check - daemon.controller.power_on.assert_called_once() + item = _dequeue_item(daemon.action_queue) + assert item.action == bmcctld.ACTION_POWER_ON def test_reboot_cause_exception_falls_back_to_cold_boot(self, chassis): """If chassis.get_reboot_cause() raises, fall back to cold-boot behavior (apply delay).""" @@ -1407,17 +5921,322 @@ def _make_daemon(self, chassis): return_value=bmcctld.ADMIN_UP) return daemon + @pytest.fixture + def lifecycle(self, chassis, monkeypatch): + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) + daemon = self._make_daemon(chassis) + chassis.switch_host.set_admin_state = MagicMock() + chassis.switch_host.do_power_cycle = MagicMock() + daemon.controller.write_operation_result = MagicMock( + wraps=daemon.controller.write_operation_result) + daemon._run_action_loop = MagicMock(wraps=daemon._run_action_loop) + event_thread = MagicMock(name="event_thread") + event_thread.start.side_effect = daemon.event_handler._subscription_ready.set + worker = MagicMock(name="operation_thread") + worker.is_alive.return_value = True + worker.start.side_effect = daemon.stop_event.set + + def join_worker(timeout): + assert daemon.operation_runner.current.cancel.is_set() + + def join_events(timeout): + assert daemon.stop_event.is_set() + operation = daemon.operation_runner.current + if operation is not None and worker.is_alive(): + assert operation.cancel.is_set() + + worker.join.side_effect = join_worker + event_thread.join.side_effect = join_events + + def make_thread(*, target, name, daemon, args=()): + if name == "bmcctld-events": + return event_thread + assert name == "bmcctld-op" + return worker + + monkeypatch.setattr(bmcctld.threading, "Thread", make_thread) + monkeypatch.setattr(bmcctld, "BmcctldDaemon", MagicMock(return_value=daemon)) + yield daemon, event_thread, worker + chassis.switch_host.set_admin_state.assert_not_called() + chassis.switch_host.do_power_cycle.assert_not_called() + + @pytest.mark.parametrize("completion", ["alive", "before_stop", "worker_join", "event_join"]) + def test_main_shutdown_joins_worker_once(self, lifecycle, completion): + daemon, event_thread, worker = lifecycle + first_callback, joined_callback = MagicMock(), MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_RESTART, "test", 4, + on_complete=first_callback)) + operation = None + + def finish(): + worker.is_alive.return_value = False + operation.outcome = (bmcctld.OP_RESULT_PREEMPTED, "-", False) + + def start(): + nonlocal operation + operation = daemon.operation_runner.current + operation.joined_callbacks.append((joined_callback, None)) + daemon.stop_event.set() + if completion == "before_stop": + finish() + + worker.start.side_effect = start + if completion == "worker_join": + def finish_on_worker_join(timeout): + assert operation.cancel.is_set() + finish() + worker.join.side_effect = finish_on_worker_join + elif completion == "event_join": + def finish_on_event_join(timeout): + assert daemon.stop_event.is_set() + assert operation.cancel.is_set() + finish() + event_thread.join.side_effect = finish_on_event_join + + assert bmcctld.main() == bmcctld.exit_code + + worker.start.assert_called_once() + assert worker.join.call_args_list == ( + [] if completion == "before_stop" else [call(timeout=5)]) + event_thread.join.assert_called_once_with(timeout=5) + assert daemon.action_queue.empty() + if completion in ("before_stop", "worker_join"): + # A worker the stop itself cancelled is recorded as abandoned; + # one that finished before the stop keeps its own result. + expected = (bmcctld.OP_RESULT_ABANDONED if completion == "worker_join" + else bmcctld.OP_RESULT_PREEMPTED) + assert daemon.operation_runner.current is None + daemon.controller.write_operation_result.assert_called_once_with( + expected, "-") + for callback in (first_callback, joined_callback): + callback.assert_called_once_with(False, expected) + daemon.operation_runner.stop() + daemon.controller.write_operation_result.assert_called_once() + first_callback.assert_called_once() + joined_callback.assert_called_once() + else: + assert daemon.operation_runner.current is operation + daemon.controller.write_operation_result.assert_not_called() + first_callback.assert_not_called() + joined_callback.assert_not_called() + state = dict(daemon.controller.host_state_table.get(bmcctld.HOST_STATE_KEY)[1]) + assert state[bmcctld.FIELD_OP_RESULT] == "-" + + @pytest.mark.parametrize("startup_drain", [False, True]) + @pytest.mark.parametrize("completion", ["alive", "displacement", "cleanup"]) + def test_displacement_stop_reaches_run_cleanup( + self, lifecycle, chassis, monkeypatch, startup_drain, completion): + daemon, event_thread, worker = lifecycle + callback, joined_callback, successor_callback = MagicMock(), MagicMock(), MagicMock() + runner = daemon.operation_runner + runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_RESTART, "running", 4, on_complete=callback)) + successor = bmcctld.ActionItem( + bmcctld.ACTION_POWER_OFF, "next", 2, on_complete=successor_callback) + entry = (successor.priority, 42, successor) + operation = None + if startup_drain: + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + chassis.set_liquid_cooled(True) + chassis.set_reboot_cause(bmcctld.ChassisBase.REBOOT_CAUSE_POWER_LOSS) + daemon.policy_reader.get_power_on_delay = MagicMock(return_value=20) + monkeypatch.setattr(bmcctld.time, "clock_gettime", lambda clock: 10) + monkeypatch.setattr(bmcctld.time, "monotonic", lambda: 0) + + def start_worker(): + nonlocal operation + operation = runner.current + operation.joined_callbacks.append((joined_callback, None)) + if completion != "alive": + operation.outcome = (bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-", True) + daemon.action_queue.put(entry) + + def join_worker(timeout): + assert operation.cancel.is_set() + if not daemon.stop_event.is_set(): + assert timeout == 1 + daemon.stop_event.set() + if completion == "displacement": + worker.is_alive.return_value = False + else: + assert timeout == 5 + if completion == "cleanup": + worker.is_alive.return_value = False + + worker.start.side_effect = start_worker + worker.join.side_effect = join_worker + + assert bmcctld.main() == bmcctld.exit_code + + worker.start.assert_called_once() + assert worker.join.call_args_list == ( + [call(timeout=1)] if completion == "displacement" else + [call(timeout=1), call(timeout=5)]) + event_thread.join.assert_called_once_with(timeout=5) + assert daemon._run_action_loop.call_count == (0 if startup_drain else 1) + assert daemon.action_queue.get_nowait() is entry + assert daemon.action_queue.empty() + successor_callback.assert_not_called() + if completion == "alive": + assert runner.current is operation + assert operation.outcome is None + daemon.controller.write_operation_result.assert_not_called() + callback.assert_not_called() + joined_callback.assert_not_called() + else: + assert runner.current is None + daemon.controller.write_operation_result.assert_called_once_with( + bmcctld.OP_RESULT_SUCCESS_GRACEFUL, "-") + callback.assert_called_once_with(True, bmcctld.OP_RESULT_SUCCESS_GRACEFUL) + joined_callback.assert_called_once_with(True, bmcctld.OP_RESULT_SUCCESS_GRACEFUL) + + @pytest.mark.parametrize("fail_after_admission", [False, True]) + def test_startup_drain_cleans_up_before_event_join( + self, lifecycle, chassis, monkeypatch, fail_after_admission): + daemon, event_thread, worker = lifecycle + chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) + chassis.set_liquid_cooled(True) + chassis.set_reboot_cause(bmcctld.ChassisBase.REBOOT_CAUSE_POWER_LOSS) + daemon.policy_reader.get_power_on_delay = MagicMock(return_value=20) + monkeypatch.setattr(bmcctld.time, "clock_gettime", lambda clock: 10) + monkeypatch.setattr(bmcctld.time, "monotonic", lambda: 0) + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_RESTART, "test", 4)) + process_next = daemon.operation_runner.process_next + + def drain(timeout): + assert process_next(timeout=0) + worker.start.assert_called_once() + if fail_after_admission: + raise RuntimeError("drain failed") + return True + + daemon.operation_runner.process_next = MagicMock(side_effect=drain) + if fail_after_admission: + with pytest.raises(RuntimeError, match="drain failed"): + bmcctld.main() + else: + assert bmcctld.main() == bmcctld.exit_code + daemon.policy_reader.get_power_on_delay.assert_called_once() + daemon.operation_runner.process_next.assert_called_once_with(timeout=1.0) + daemon._run_action_loop.assert_not_called() + worker.join.assert_called_once_with(timeout=5) + event_thread.join.assert_called_once_with(timeout=5) + assert daemon.operation_runner.current.cancel.is_set() + assert daemon.action_queue.empty() + daemon.controller.write_operation_result.assert_not_called() + + def test_action_loop_error_after_admission_still_cleans_up(self, lifecycle): + daemon, event_thread, worker = lifecycle + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_RESTART, "test", 4)) + process_next = daemon.operation_runner.process_next + + def admit_then_fail(timeout): + assert process_next(timeout=0) + worker.start.assert_called_once() + raise RuntimeError("action loop failed") + + daemon.operation_runner.process_next = MagicMock(side_effect=admit_then_fail) + with pytest.raises(RuntimeError, match="action loop failed"): + bmcctld.main() + daemon._run_action_loop.assert_called_once() + worker.join.assert_called_once_with(timeout=5) + event_thread.join.assert_called_once_with(timeout=5) + assert daemon.stop_event.is_set() + assert daemon.operation_runner.current.cancel.is_set() + assert daemon.action_queue.empty() + daemon.controller.write_operation_result.assert_not_called() + + def test_stop_time_record_failure_still_joins_event_thread(self, lifecycle): + daemon, event_thread, worker = lifecycle + callback, joined_callback = MagicMock(), MagicMock() + daemon.operation_runner.enqueue(bmcctld.ActionItem( + bmcctld.ACTION_GRACEFUL_RESTART, "test", 4, on_complete=callback)) + + def finish(timeout): + operation = daemon.operation_runner.current + assert operation.cancel.is_set() + operation.joined_callbacks.append((joined_callback, None)) + operation.outcome = (bmcctld.OP_RESULT_PREEMPTED, "-", False) + worker.is_alive.return_value = False + + worker.join.side_effect = finish + daemon.controller.write_operation_result.side_effect = RuntimeError("record failed") + with pytest.raises(RuntimeError, match="record failed"): + bmcctld.main() + worker.join.assert_called_once_with(timeout=5) + event_thread.join.assert_called_once_with(timeout=5) + assert daemon.stop_event.is_set() + assert daemon.operation_runner.current is not None + daemon.controller.write_operation_result.assert_called_once_with( + bmcctld.OP_RESULT_ABANDONED, "-") + callback.assert_not_called() + joined_callback.assert_not_called() + + @pytest.mark.parametrize("subscription_error", [False, True]) + def test_subscription_exit_cleans_up_without_worker(self, lifecycle, subscription_error): + daemon, event_thread, worker = lifecycle + wait = MagicMock(return_value=False) + if subscription_error: + wait.side_effect = RuntimeError("subscription failed") + daemon.event_handler.wait_for_event_subscriptions = wait + if subscription_error: + with pytest.raises(RuntimeError, match="subscription failed"): + bmcctld.main() + else: + assert bmcctld.main() == bmcctld.exit_code + event_thread.join.assert_called_once_with(timeout=5) + assert daemon.stop_event.is_set() + assert daemon.operation_runner.current is None + worker.start.assert_not_called() + worker.join.assert_not_called() + daemon._run_action_loop.assert_not_called() + daemon.controller.write_operation_result.assert_not_called() + + @pytest.mark.parametrize("failure", ["init", "seed", "thread_start"]) + def test_failure_before_event_start_has_no_worker_or_join(self, lifecycle, failure): + daemon, event_thread, worker = lifecycle + target = { + "init": (daemon.controller, "init_host_state"), + "seed": (daemon.event_handler, "seed_chassis_module_admin_status"), + "thread_start": (event_thread, "start"), + }[failure] + with patch.object(*target, side_effect=RuntimeError("startup failed")): + with pytest.raises(RuntimeError, match="startup failed"): + bmcctld.main() + event_thread.join.assert_not_called() + worker.start.assert_not_called() + worker.join.assert_not_called() + assert daemon.operation_runner.current is None + daemon._run_action_loop.assert_not_called() + daemon.controller.write_operation_result.assert_not_called() + + def test_main_stop_without_current_worker(self, lifecycle): + daemon, event_thread, worker = lifecycle + daemon.stop_event.set() + assert bmcctld.main() == bmcctld.exit_code + event_thread.join.assert_called_once_with(timeout=5) + worker.start.assert_not_called() + worker.join.assert_not_called() + assert daemon.operation_runner.current is None + daemon.controller.write_operation_result.assert_not_called() + def test_run_not_liquid_cooled_powers_on_immediately(self, chassis): """Non-liquid-cooled + admin up: power_on is called immediately.""" chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_OFFLINE) daemon = self._make_daemon(chassis) - daemon.controller.power_on = MagicMock(return_value=True) daemon._run_action_loop = MagicMock() - daemon.event_handler.run_event_loop = MagicMock() + daemon.event_handler.run_event_loop = MagicMock( + side_effect=daemon.event_handler._subscription_ready.set) chassis.set_liquid_cooled(False) result = daemon.run() assert result is False - daemon.controller.power_on.assert_called_once() + item = _dequeue_item(daemon.action_queue) + assert item.action == bmcctld.ACTION_POWER_ON + assert item.event_desc == "STARTUP" + assert item.startup is True daemon._run_action_loop.assert_called_once() def test_run_not_liquid_cooled_skips_power_on_when_admin_down(self, chassis): @@ -1428,9 +6247,11 @@ def test_run_not_liquid_cooled_skips_power_on_when_admin_down(self, chassis): return_value=bmcctld.ADMIN_DOWN) daemon.controller.power_on = MagicMock(return_value=True) daemon._run_action_loop = MagicMock() - daemon.event_handler.run_event_loop = MagicMock() + daemon.event_handler.run_event_loop = MagicMock( + side_effect=daemon.event_handler._subscription_ready.set) chassis.set_liquid_cooled(False) daemon.run() + assert daemon.action_queue.empty() daemon.controller.power_on.assert_not_called() daemon._run_action_loop.assert_called_once() @@ -1441,7 +6262,8 @@ def test_run_not_liquid_cooled_skips_initial_sequence_but_starts_event_thread(se daemon.controller.power_on = MagicMock(return_value=True) daemon._run_action_loop = MagicMock() daemon._initial_power_on_sequence = MagicMock() - daemon.event_handler.run_event_loop = MagicMock() + daemon.event_handler.run_event_loop = MagicMock( + side_effect=daemon.event_handler._subscription_ready.set) chassis.set_liquid_cooled(False) daemon.run() daemon._initial_power_on_sequence.assert_not_called() @@ -1455,11 +6277,13 @@ def test_run_daemon_restart_skips_boot_sequence(self, chassis): daemon.controller.power_on = MagicMock() daemon.controller.init_host_state = MagicMock() daemon._run_action_loop = MagicMock() - daemon.event_handler.run_event_loop = MagicMock() + daemon.event_handler.run_event_loop = MagicMock( + side_effect=daemon.event_handler._subscription_ready.set) chassis.set_liquid_cooled(True) result = daemon.run() assert result is False daemon._initial_power_on_sequence.assert_not_called() + assert daemon.action_queue.empty() daemon.controller.power_on.assert_not_called() daemon.controller.init_host_state.assert_called_once() daemon._run_action_loop.assert_called_once() @@ -1477,6 +6301,7 @@ def track_seed(): call_order.append("seed") def track_event_loop(): + daemon.event_handler._subscription_ready.set() call_order.append("event_loop") daemon.controller.init_host_state = MagicMock(side_effect=track_init) @@ -1494,7 +6319,8 @@ def test_run_liquid_cooled_runs_full_sequence(self, chassis): daemon = self._make_daemon(chassis) daemon._initial_power_on_sequence = MagicMock() daemon._run_action_loop = MagicMock() - daemon.event_handler.run_event_loop = MagicMock() + daemon.event_handler.run_event_loop = MagicMock( + side_effect=daemon.event_handler._subscription_ready.set) # Set stop_event so _run_action_loop returns without looping daemon._initial_power_on_sequence.side_effect = lambda: daemon.stop_event.set() chassis.set_liquid_cooled(True) @@ -1502,6 +6328,17 @@ def test_run_liquid_cooled_runs_full_sequence(self, chassis): assert result is False daemon._initial_power_on_sequence.assert_called_once() + def test_run_propagates_event_subscription_setup_error(self, chassis): + daemon = self._make_daemon(chassis) + daemon.event_handler._create_event_subscriptions = MagicMock( + side_effect=RuntimeError("subscription setup failed")) + daemon._run_action_loop = MagicMock() + + with pytest.raises(RuntimeError, match="subscription setup failed"): + daemon.run() + + daemon._run_action_loop.assert_not_called() + # -------------------------------------------------------------------------- # Tests: ChassisModuleInfo — CHASSIS_MODULE_TABLE STATE_DB integration @@ -1546,7 +6383,7 @@ def test_initialize_chassis_module_no_module(self, controller): def test_power_on_mirrors_oper_status_online(self, chassis, controller): """power_on updates oper_status=ONLINE in CHASSIS_MODULE_TABLE.""" - controller.power_on() + controller.power_on(_make_operation(bmcctld.ACTION_POWER_ON)) result = controller.chassis_module_info_table.get(bmcctld.SWITCH_HOST_MODULE_KEY) assert result[0] is True info = dict(result[1]) @@ -1555,7 +6392,7 @@ def test_power_on_mirrors_oper_status_online(self, chassis, controller): def test_power_off_mirrors_oper_status_offline(self, chassis, controller): """power_off updates oper_status=OFFLINE in CHASSIS_MODULE_TABLE.""" chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) - controller.power_off() + controller.power_off(_make_operation(bmcctld.ACTION_POWER_OFF)) result = controller.chassis_module_info_table.get(bmcctld.SWITCH_HOST_MODULE_KEY) assert result[0] is True info = dict(result[1]) @@ -1575,7 +6412,7 @@ def test_initialize_then_power_off_preserves_static_fields(self, chassis, contro chassis.switch_host.set_oper_status(MockModule.MODULE_STATUS_ONLINE) controller.initialize_chassis_module(bmcctld.ADMIN_UP) # Now power off — oper_status should update but name/serial/etc. must survive - controller.power_off() + controller.power_off(_make_operation(bmcctld.ACTION_POWER_OFF)) result = controller.chassis_module_info_table.get(bmcctld.SWITCH_HOST_MODULE_KEY) info = dict(result[1]) assert info[bmcctld.CHASSIS_MODULE_INFO_NAME_FIELD] == "SWITCH-HOST" @@ -1634,16 +6471,17 @@ def test_initialize_chassis_module_seeds_config_db_defaults(self, chassis, contr cfg = dict(result[1]) assert cfg[bmcctld.CHASSIS_MODULE_INFO_ADMIN_STATUS_FIELD] == bmcctld.ADMIN_DOWN assert cfg[bmcctld.FIELD_POWER_ON_DELAY] == str(bmcctld.DEFAULT_POWER_ON_DELAY_SECS) - assert cfg[bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT] == str(bmcctld.DEFAULT_SHUTDOWN_DELAY_SECS) + assert cfg[bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT] == "120" - def test_initialize_chassis_module_preserves_operator_config(self, chassis, controller): + @pytest.mark.parametrize("seconds", ["0", "90", "300"]) + def test_initialize_chassis_module_preserves_operator_config(self, chassis, controller, seconds): """Existing operator CONFIG_DB entry must not be clobbered by daemon startup.""" controller.chassis_module_config_table.set( bmcctld.SWITCH_HOST_MODULE_KEY, FieldValuePairs([ (bmcctld.CHASSIS_MODULE_INFO_ADMIN_STATUS_FIELD, bmcctld.ADMIN_UP), (bmcctld.FIELD_POWER_ON_DELAY, "45"), - (bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT, "90"), + (bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT, seconds), ]), ) controller.initialize_chassis_module(bmcctld.ADMIN_DOWN) @@ -1651,7 +6489,7 @@ def test_initialize_chassis_module_preserves_operator_config(self, chassis, cont cfg = dict(result[1]) assert cfg[bmcctld.CHASSIS_MODULE_INFO_ADMIN_STATUS_FIELD] == bmcctld.ADMIN_UP assert cfg[bmcctld.FIELD_POWER_ON_DELAY] == "45" - assert cfg[bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT] == "90" + assert cfg[bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT] == seconds def test_daemon_init_air_cooled_defaults_admin_up(self, chassis): """Air-cooled boxes default CONFIG_DB admin_status=up when no operator entry exists.""" @@ -1712,3 +6550,342 @@ def test_module_loggers_target_expected_files(self): """_daemon_logger is syslog-only; _event_logger keeps the host file.""" assert bmcctld._daemon_logger._file_logger.name.endswith("syslog_only") assert bmcctld._event_logger._file_logger.name.endswith("event_log") + + +class TestDatabaseThreadOwnership: + + def test_subscription_wait_returns_when_daemon_stops(self, event_handler): + event_handler.stop_event.clear() + result = [] + waiter = threading.Thread( + target=lambda: result.append( + event_handler.wait_for_event_subscriptions())) + waiter.start() + event_handler.stop_event.set() + waiter.join(2) + + assert not waiter.is_alive() + assert result == [False] + + def test_each_execution_thread_owns_its_database_objects( + self, monkeypatch, chassis): + import weakref + + store = {} + created = [] + connection_records = [] + connection_sequence = itertools.count() + errors = [] + admission_thread_name = threading.current_thread().name + release = threading.Event() + event_write_started = threading.Event() + worker_write_started = threading.Event() + stop_event = None + + def table_data(db_name, table_name): + return store.setdefault((db_name, table_name), {}) + + table_data("STATE_DB", bmcctld.RACK_MANAGER_COMMAND_TABLE)["CMD_THREAD"] = { + bmcctld.FIELD_COMMAND: bmcctld.CMD_POWER_OFF, + bmcctld.FIELD_STATUS: bmcctld.CMD_STATUS_PENDING, + } + table_data("CONFIG_DB", bmcctld.CHASSIS_MODULE_TABLE)[ + bmcctld.SWITCH_HOST_MODULE_KEY] = { + bmcctld.FIELD_ADMIN_STATUS: bmcctld.ADMIN_UP, + bmcctld.FIELD_GRACEFUL_SHUTDOWN_TIMEOUT: "10", + } + table_data("STATE_DB", bmcctld.SYSTEM_LEAK_STATUS_TABLE)[ + bmcctld.SYSTEM_LEAK_STATUS_KEY] = { + bmcctld.FIELD_DEVICE_LEAK_STATUS: "NORMAL", + } + + class OwnedObject: + def __init__(self, kind, retain=True): + self.kind = kind + self.owner_id = threading.get_ident() + self.owner_name = threading.current_thread().name + self._active = threading.Lock() + if retain: + created.append(self) + + def check_owner(self): + assert threading.get_ident() == self.owner_id, ( + "{} created by {} used by {}".format( + self.kind, self.owner_name, + threading.current_thread().name)) + + def enter(self): + self.check_owner() + assert self._active.acquire(False), ( + "overlapping use of {} owned by {}".format( + self.kind, self.owner_name)) + + def leave(self): + self._active.release() + + class Connection(OwnedObject): + def __init__(self, db_name): + self.db_name = db_name + self.token = next(connection_sequence) + super().__init__("{} connector".format(db_name), retain=False) + connection_records.append( + (self.token, self.db_name, self.owner_name)) + + class StrictTable(OwnedObject): + def __init__(self, db, table_name, kind="table", retain_db=True): + db.check_owner() + self._db = db if retain_db else weakref.ref(db) + self.connector_id = db.token + self.table_name = table_name + self._waited = False + self._active_db = None + super().__init__("{} {}".format(table_name, kind)) + + def connection(self): + db = self._db() if isinstance( + self._db, weakref.ReferenceType) else self._db + assert db is not None, ( + "{} lost its source connector".format(self.kind)) + return db + + def enter(self): + db = self.connection() + db.enter() + try: + super().enter() + except Exception: + db.leave() + raise + self._active_db = db + + def leave(self): + super().leave() + self._active_db.leave() + self._active_db = None + + def _maybe_wait(self, method): + if self._waited or method != "set": + return + role = threading.current_thread().name + if role == "db-owner-event" and \ + self.table_name == bmcctld.RACK_MANAGER_COMMAND_TABLE: + started = event_write_started + elif role == "db-owner-worker" and \ + self.table_name == bmcctld.HOST_STATE_TABLE: + started = worker_write_started + else: + return + self._waited = True + started.set() + assert release.wait(3), "timed out waiting for concurrent DB access" + + def set(self, key, fvs): + self.enter() + try: + self._maybe_wait("set") + values = fvs.fv_dict if hasattr(fvs, "fv_dict") else dict(fvs) + table_data(self.connection().db_name, self.table_name).setdefault( + key, {}).update(values) + finally: + self.leave() + + def get(self, key): + self.enter() + try: + value = table_data( + self.connection().db_name, self.table_name).get(key) + return [value is not None, list(value.items()) if value else []] + finally: + self.leave() + + def getKeys(self): + self.enter() + try: + return list(table_data( + self.connection().db_name, self.table_name)) + finally: + self.leave() + + class StrictSubscriber(StrictTable): + def __init__(self, db, table_name): + super().__init__( + db, table_name, "subscriber", retain_db=False) + + def getFd(self): + self.check_owner() + return id(self) + + def pop(self): + self.enter() + try: + rows = table_data( + self.connection().db_name, self.table_name) + key = next(iter(rows)) + return key, "SET", list(rows[key].items()) + finally: + self.leave() + + class StrictSelect(OwnedObject): + OBJECT = 0 + TIMEOUT = 1 + + def __init__(self): + self.selectables = [] + self.event_index = 0 + super().__init__("select") + + def addSelectable(self, selectable): + self.enter() + try: + selectable.check_owner() + self.selectables.append(selectable) + finally: + self.leave() + + def select(self, timeout=-1, interrupt_on_signal=False): + self.enter() + try: + event_tables = ( + bmcctld.RACK_MANAGER_COMMAND_TABLE, + bmcctld.CHASSIS_MODULE_TABLE, + ) + if self.event_index == len(event_tables): + return self.TIMEOUT, None + table_name = event_tables[self.event_index] + self.event_index += 1 + selected = next( + item for item in self.selectables + if item.table_name == table_name) + if self.event_index == len(event_tables): + stop_event.set() + return self.OBJECT, SelectedObject(selected) + finally: + self.leave() + + class SelectedObject(OwnedObject): + def __init__(self, subscriber): + self.fd = subscriber.getFd() + super().__init__("selected object") + + def getFd(self): + self.check_owner() + return self.fd + + monkeypatch.setattr( + bmcctld.daemon_base, "db_connect", lambda db_name: Connection(db_name)) + monkeypatch.setattr( + bmcctld.swsscommon, "Table", + lambda db, table_name: StrictTable(db, table_name)) + monkeypatch.setattr( + bmcctld.swsscommon, "SubscriberStateTable", StrictSubscriber) + monkeypatch.setattr(bmcctld.swsscommon, "Select", StrictSelect) + + with patch('sonic_platform.platform.Platform') as mock_platform: + mock_platform.return_value.get_chassis.return_value = chassis + daemon = bmcctld.BmcctldDaemon(bmcctld.SYSLOG_IDENTIFIER) + controller = daemon.controller + policy_reader = daemon.policy_reader + critical_checker = daemon.critical_event_checker + action_queue = daemon.action_queue + event_handler = daemon.event_handler + stop_event = daemon.stop_event + + def capture_errors(function): + try: + function() + except Exception as error: + errors.append((threading.current_thread().name, error)) + + def event_access(): + event_handler.run_event_loop() + controller.get_db_device_status() + policy_reader.get_graceful_shutdown_timeout() + policy_reader.get_gnoi_cert_paths() + critical_checker.get_system_leak_status() + + def worker_access(): + controller._update_host_state( + bmcctld.SWITCH_HOST_POWERING_OFF, + bmcctld.SWITCH_HOST_OFFLINE) + policy_reader.get_graceful_shutdown_timeout() + policy_reader.get_gnoi_cert_paths() + critical_checker.get_system_leak_status() + + event_thread = threading.Thread( + target=capture_errors, args=(event_access,), name="db-owner-event") + worker_thread = threading.Thread( + target=capture_errors, args=(worker_access,), name="db-owner-worker") + event_thread.start() + worker_thread.start() + + both_started = event_write_started.wait(2) and worker_write_started.wait(2) + if both_started: + controller.write_operation_start(TEST_UUID4, "test") + policy_reader.get_graceful_shutdown_timeout() + policy_reader.get_gnoi_cert_paths() + critical_checker.get_system_leak_status() + event_handler._set_cmd_request_id("CMD_THREAD", TEST_UUID4) + release.set() + event_thread.join(3) + worker_thread.join(3) + + assert both_started, errors + assert not event_thread.is_alive() + assert not worker_thread.is_alive() + assert errors == [] + + item = _dequeue_item(action_queue) + item.on_complete(True, "") + command = table_data( + "STATE_DB", bmcctld.RACK_MANAGER_COMMAND_TABLE)["CMD_THREAD"] + assert command[bmcctld.FIELD_REQUEST_ID] == TEST_UUID4 + assert command[bmcctld.FIELD_STATUS] == bmcctld.CMD_STATUS_DONE + + for role in ( + admission_thread_name, "db-owner-event", "db-owner-worker"): + assert any(obj.table_name == bmcctld.BMC_GNOI_TABLE and + obj.owner_name == role for obj in created + if isinstance(obj, StrictTable)) + assert { + db_name for _, db_name, owner_name in connection_records + if owner_name == role + } == { + "STATE_DB", "CONFIG_DB"} + + role_connector_ids = { + role: { + object_id for object_id, _, owner_name in connection_records + if owner_name == role + } + for role in ( + admission_thread_name, "db-owner-event", "db-owner-worker") + } + assert role_connector_ids[admission_thread_name].isdisjoint( + role_connector_ids["db-owner-event"]) + assert role_connector_ids[admission_thread_name].isdisjoint( + role_connector_ids["db-owner-worker"]) + assert role_connector_ids["db-owner-event"].isdisjoint( + role_connector_ids["db-owner-worker"]) + + event_command_tables = [ + obj for obj in created + if isinstance(obj, StrictTable) and + not isinstance(obj, StrictSubscriber) and + obj.owner_name == "db-owner-event" and + obj.table_name == bmcctld.RACK_MANAGER_COMMAND_TABLE] + admission_command_tables = [ + obj for obj in created + if isinstance(obj, StrictTable) and + not isinstance(obj, StrictSubscriber) and + obj.owner_name == admission_thread_name and + obj.table_name == bmcctld.RACK_MANAGER_COMMAND_TABLE] + command_subscribers = [ + obj for obj in created + if isinstance(obj, StrictSubscriber) and + obj.table_name == bmcctld.RACK_MANAGER_COMMAND_TABLE] + assert len(event_command_tables) == 1 + assert len(admission_command_tables) == 1 + assert len(command_subscribers) == 1 + assert event_command_tables[0] is not admission_command_tables[0] + assert event_command_tables[0].connector_id != \ + command_subscribers[0].connector_id