2025-02-13 Thu 02:48:05:038.132 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738719057664, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:48:30:056.285 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738719057664, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 31 without seqno , total = 0 2025-02-13 Thu 02:48:30:056.400 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:48:30:057.162 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: SANDESH: Send FAILED: 1739414910056080 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 298 dequeues = 298 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 402 dequeues = 402 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739414881038691 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739414885037631 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:48:30:057.193 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:48:30:057.209 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:48:30:057.404 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: SANDESH: Send FAILED: 1739414910056256 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 74 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 193 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 75 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 117, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 298, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:48:30:057.435 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:48:31:039.081 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:48:31:039.482 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:48:31:039.495 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:48:31:039.590 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Idle 2025-02-13 Thu 02:48:31:039.881 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:48:31:039.913 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739414911039530 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:48:31:039.923 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:48:31:039.932 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:48:31:039.983 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:48:31:040.138 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739414911039868 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 74 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 195 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 75 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 117, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 300, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:48:31:040.158 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:49:00:056.649 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:49:00:056.767 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: SANDESH: Send FAILED: 1739414940056474 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 302 dequeues = 302 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 408 dequeues = 408 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739414911039588 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739414911039967 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:49:00:056.794 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:49:00:056.806 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:49:00:056.994 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: SANDESH: Send FAILED: 1739414940056601 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 75 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 196 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 75 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 117, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 302, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:49:00:057.013 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:49:01:039.755 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:49:01:040.119 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:49:01:040.286 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:49:01:040.332 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: SANDESH: Send FAILED: 1739414941039867 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:49:01:040.345 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:49:01:040.359 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:49:01:040.622 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: SANDESH: Send FAILED: 1739414941040264 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 75 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 31 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 198 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 75 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 117, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 304, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:49:01:040.644 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:49:05:039.598 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:49:05:039.726 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: SANDESH: Send FAILED: 1739414945039540 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:49:05:039.742 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:49:30:056.911 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738719057664, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 32 without seqno , total = 0 2025-02-13 Thu 02:49:30:057.020 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:49:30:057.156 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: SANDESH: Send FAILED: 1739414970056812 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 307 dequeues = 307 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 414 dequeues = 414 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739414941039876 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739414945039580 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:49:30:057.199 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:49:30:057.216 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:49:30:057.366 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: SANDESH: Send FAILED: 1739414970056891 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 76 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 199 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 75 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 117, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 307, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:49:30:057.383 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:49:31:040.275 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:49:31:040.370 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:49:31:040.454 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:49:31:040.562 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Idle 2025-02-13 Thu 02:49:31:040.658 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:49:31:040.689 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: SANDESH: Send FAILED: 1739414971040497 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:49:31:040.699 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:49:31:040.709 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:49:31:040.760 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:49:31:040.912 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: SANDESH: Send FAILED: 1739414971040645 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 76 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 201 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 75 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 117, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 309, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:49:31:040.932 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:49:33:479.160 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:57976 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:49:33:522.986 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:57976 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:49:44:189.741 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:39888 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:49:44:234.929 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:39888 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:50:00:058.244 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:50:00:058.344 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: SANDESH: Send FAILED: 1739415000058098 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 311 dequeues = 311 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 420 dequeues = 420 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739414971040560 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739414971040758 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:50:00:058.388 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:50:00:058.399 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:50:00:058.543 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: SANDESH: Send FAILED: 1739415000058209 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 77 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 202 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 79 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 121, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 311, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:50:00:058.557 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:50:01:040.789 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:50:01:041.062 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:50:01:041.161 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:50:01:041.193 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739415001040880 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:50:01:041.203 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:50:01:041.213 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:50:01:041.349 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739415001041149 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 77 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 204 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 79 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 121, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 313, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:50:01:041.372 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:50:05:040.171 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:50:05:040.263 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739415005040121 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:50:05:040.275 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:50:30:058.880 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 33 without seqno , total = 0 2025-02-13 Thu 02:50:30:058.984 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:50:30:059.510 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: SANDESH: Send FAILED: 1739415030058751 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 316 dequeues = 316 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 426 dequeues = 426 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739415001040888 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739415005040155 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:50:30:059.568 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:50:30:059.605 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:50:30:059.853 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: SANDESH: Send FAILED: 1739415030058858 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 78 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 205 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 79 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 121, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 316, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:50:30:059.887 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:50:31:041.217 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:50:31:041.321 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:50:31:041.364 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:50:31:041.464 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Idle 2025-02-13 Thu 02:50:31:041.574 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:50:31:041.608 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: SANDESH: Send FAILED: 1739415031041402 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:50:31:041.618 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:50:31:041.628 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:50:31:041.666 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:50:31:041.805 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: SANDESH: Send FAILED: 1739415031041562 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 78 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 207 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 79 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 121, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 318, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:50:31:041.833 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:51:00:059.288 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:51:00:059.401 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: SANDESH: Send FAILED: 1739415060059085 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 320 dequeues = 320 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 432 dequeues = 432 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739415031041462 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739415031041664 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:51:00:059.418 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:51:00:059.429 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:51:00:059.575 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: SANDESH: Send FAILED: 1739415060059237 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 79 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 208 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 79 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 121, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 320, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:51:00:059.597 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:51:01:041.661 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:51:01:041.981 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:51:01:042.233 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:51:01:042.283 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: SANDESH: Send FAILED: 1739415061041758 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:51:01:042.297 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:51:01:042.308 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:51:01:042.495 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: SANDESH: Send FAILED: 1739415061042208 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 79 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 33 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 210 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 79 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 121, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 322, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:51:01:042.519 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738736711424, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:51:05:041.766 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737662719744, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:51:05:041.877 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737662719744, Pid 1]: SANDESH: Send FAILED: 1739415065041719 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:51:05:041.890 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737662719744, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:51:30:059.748 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 34 without seqno , total = 0 2025-02-13 Thu 02:51:30:059.836 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:51:30:059.974 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: SANDESH: Send FAILED: 1739415090059459 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 325 dequeues = 325 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 438 dequeues = 438 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739415061041767 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739415065041747 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:51:30:059.993 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:51:30:060.004 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:51:30:060.183 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: SANDESH: Send FAILED: 1739415090059729 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 80 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 211 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 79 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 121, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 325, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:51:30:060.210 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:51:31:042.169 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:51:31:042.611 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:51:31:042.628 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:51:31:042.728 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Idle 2025-02-13 Thu 02:51:31:042.821 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:51:31:042.852 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: SANDESH: Send FAILED: 1739415091042664 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:51:31:042.863 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:51:31:042.873 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:51:31:042.908 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:51:31:043.132 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: SANDESH: Send FAILED: 1739415091042808 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 80 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 213 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 79 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 121, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 327, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:51:31:043.158 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738069567232, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:51:34:458.283 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:59938 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:51:34:502.929 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:59938 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:51:45:542.401 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:58188 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:51:45:586.975 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:58188 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:52:00:060.257 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:52:00:060.374 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: SANDESH: Send FAILED: 1739415120059925 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 329 dequeues = 329 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 444 dequeues = 444 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739415091042726 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739415091042906 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:52:00:060.391 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:52:00:060.415 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:52:00:060.574 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: SANDESH: Send FAILED: 1739415120060208 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 81 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 214 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 83 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 125, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 329, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:52:00:060.599 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738749306624, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:52:01:042.902 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:52:01:043.210 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:52:01:043.318 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:52:01:043.351 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: SANDESH: Send FAILED: 1739415121043022 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:52:01:043.372 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:52:01:043.382 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:52:01:043.522 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: SANDESH: Send FAILED: 1739415121043304 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 81 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 34 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 216 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 83 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 125, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 331, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:52:01:043.537 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:52:05:042.470 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:52:05:042.577 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: SANDESH: Send FAILED: 1739415125042406 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:52:05:042.590 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738090559232, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:52:30:060.544 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737654322944, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 35 without seqno , total = 0 2025-02-13 Thu 02:52:30:060.626 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:52:30:060.762 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: SANDESH: Send FAILED: 1739415150060408 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 334 dequeues = 334 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 450 dequeues = 450 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739415121043030 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739415125042453 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:52:30:060.791 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:52:30:060.803 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:52:30:060.944 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: SANDESH: Send FAILED: 1739415150060520 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 82 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 217 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 83 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 125, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 334, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:52:30:060.980 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:52:31:043.361 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:52:31:043.476 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:52:31:043.520 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:52:31:043.618 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: Idle 2025-02-13 Thu 02:52:31:043.737 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:52:31:043.780 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: SANDESH: Send FAILED: 1739415151043557 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:52:31:043.790 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:52:31:043.800 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:52:31:043.809 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:52:31:043.974 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: SANDESH: Send FAILED: 1739415151043724 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 82 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 219 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 83 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 125, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 336, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:52:31:043.993 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737671116544, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:53:00:060.927 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:53:00:061.090 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: SANDESH: Send FAILED: 1739415180060735 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 338 dequeues = 338 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 456 dequeues = 456 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739415151043616 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739415151043808 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:53:00:061.126 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:53:00:061.141 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:53:00:061.368 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: SANDESH: Send FAILED: 1739415180060890 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 83 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 220 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 83 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 125, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 338, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:53:00:061.389 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:53:01:043.760 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:53:01:044.082 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:53:01:044.197 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:53:01:044.230 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: SANDESH: Send FAILED: 1739415181043857 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:53:01:044.240 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:53:01:044.250 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:53:01:044.399 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: SANDESH: Send FAILED: 1739415181044182 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 83 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 35 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 222 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 83 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 125, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 340, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:53:01:044.425 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:53:05:043.109 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738732513024, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:53:05:043.213 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738732513024, Pid 1]: SANDESH: Send FAILED: 1739415185043053 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:53:05:043.226 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738732513024, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:53:30:061.242 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 36 without seqno , total = 0 2025-02-13 Thu 02:53:30:061.320 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738082162432, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:53:30:061.447 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738082162432, Pid 1]: SANDESH: Send FAILED: 1739415210061133 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 343 dequeues = 343 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 462 dequeues = 462 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739415181043864 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739415185043092 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:53:30:061.464 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738082162432, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:53:30:061.488 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738082162432, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:53:30:061.624 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738082162432, Pid 1]: SANDESH: Send FAILED: 1739415210061224 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 84 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 223 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 83 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 125, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 343, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:53:30:061.639 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738082162432, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:53:31:044.246 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:53:31:044.315 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:53:31:044.326 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:53:31:044.468 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Idle 2025-02-13 Thu 02:53:31:044.565 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:53:31:044.596 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: SANDESH: Send FAILED: 1739415211044402 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:53:31:044.605 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:53:31:044.615 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:53:31:044.625 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:53:31:044.780 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: SANDESH: Send FAILED: 1739415211044553 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 84 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 225 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 83 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 125, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 345, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:53:31:044.806 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:54:00:061.566 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:54:00:061.897 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739415240061416 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 347 dequeues = 347 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 468 dequeues = 468 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739415211044464 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739415211044623 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:54:00:061.922 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:54:00:061.940 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:54:00:062.186 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739415240061517 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 85 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 226 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 83 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 125, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 347, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:54:00:062.217 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:54:00:218.287 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:56374 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:54:00:263.216 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:56374 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:54:01:044.626 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738757703424, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:54:01:044.903 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738757703424, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:54:01:045.024 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738757703424, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:54:01:045.063 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738757703424, Pid 1]: SANDESH: Send FAILED: 1739415241044717 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:54:01:045.076 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738757703424, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:54:01:045.090 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738757703424, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:54:01:045.244 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738757703424, Pid 1]: SANDESH: Send FAILED: 1739415241045011 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 85 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 228 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 85 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 127, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 349, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:54:01:045.270 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738757703424, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:54:05:044.289 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:54:05:044.385 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739415245043836 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:54:05:044.412 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:54:30:061.814 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738757703424, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 37 without seqno , total = 0 2025-02-13 Thu 02:54:30:061.903 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:54:30:062.081 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: SANDESH: Send FAILED: 1739415270061724 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 352 dequeues = 352 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 474 dequeues = 474 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739415241044725 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739415245043895 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:54:30:062.259 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:54:30:062.275 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:54:30:062.415 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: SANDESH: Send FAILED: 1739415270061797 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 86 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 229 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 85 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 127, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 352, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:54:30:062.444 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:54:31:045.080 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738061170432, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:54:31:045.182 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:54:31:045.223 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:54:31:045.322 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: Idle 2025-02-13 Thu 02:54:31:045.431 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:54:31:045.462 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: SANDESH: Send FAILED: 1739415271045261 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:54:31:045.473 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:54:31:045.483 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:54:31:045.520 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:54:31:045.660 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: SANDESH: Send FAILED: 1739415271045418 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 86 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 231 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 85 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 127, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 354, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:54:31:045.685 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737658521344, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:55:00:062.213 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737662719744, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:55:00:062.323 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737662719744, Pid 1]: SANDESH: Send FAILED: 1739415300062050 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 356 dequeues = 356 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 480 dequeues = 480 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739415271045320 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739415271045518 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:55:00:062.348 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737662719744, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:55:00:062.360 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737662719744, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:55:00:062.516 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737662719744, Pid 1]: SANDESH: Send FAILED: 1739415300062168 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 87 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 232 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 85 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 127, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 356, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:55:00:062.532 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737662719744, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:55:01:045.494 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:55:01:045.754 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:55:01:045.868 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:55:01:045.901 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: SANDESH: Send FAILED: 1739415301045582 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:55:01:045.922 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:55:01:045.933 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:55:01:046.198 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: SANDESH: Send FAILED: 1739415301045849 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 87 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 37 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 234 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 85 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 127, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 358, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:55:01:046.223 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:55:05:044.942 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:55:05:045.059 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: SANDESH: Send FAILED: 1739415305044867 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:55:05:045.073 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738094757632, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:55:06:946.080 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:53348 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:55:06:991.157 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:53348 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:55:30:062.541 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 38 without seqno , total = 0 2025-02-13 Thu 02:55:30:062.641 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:55:30:062.782 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: SANDESH: Send FAILED: 1739415330062395 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 361 dequeues = 361 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 486 dequeues = 486 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739415301045592 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739415305044925 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:55:30:062.820 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:55:30:062.833 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:55:30:062.999 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: SANDESH: Send FAILED: 1739415330062521 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 88 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 235 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 87 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 129, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 361, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:55:30:063.017 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:55:31:045.908 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:55:31:046.062 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:55:31:046.104 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:55:31:046.233 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: Idle 2025-02-13 Thu 02:55:31:046.344 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:55:31:046.376 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: SANDESH: Send FAILED: 1739415331046157 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:55:31:046.396 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:55:31:046.406 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:55:31:046.443 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:55:31:046.581 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: SANDESH: Send FAILED: 1739415331046330 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 88 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 237 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 87 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 129, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 363, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:55:31:046.605 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738753505024, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:56:00:062.916 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:56:00:063.054 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: SANDESH: Send FAILED: 1739415360062763 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 365 dequeues = 365 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 492 dequeues = 492 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739415331046229 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739415331046441 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:56:00:063.072 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:56:00:063.093 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:56:00:063.253 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: SANDESH: Send FAILED: 1739415360062868 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 89 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 238 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 87 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 129, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 365, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:56:00:063.269 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:56:01:046.385 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738103150336, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:56:01:046.652 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738103150336, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:56:01:046.760 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738103150336, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:56:01:046.799 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738103150336, Pid 1]: SANDESH: Send FAILED: 1739415361046467 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:56:01:046.813 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738103150336, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:56:01:046.826 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738103150336, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:56:01:046.999 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738103150336, Pid 1]: SANDESH: Send FAILED: 1739415361046748 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 89 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 38 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 240 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 87 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 129, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 367, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:56:01:047.025 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738103150336, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:56:05:045.628 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:56:05:045.978 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: SANDESH: Send FAILED: 1739415365045575 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:56:05:046.007 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:56:06:714.646 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:54982 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:56:06:762.921 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:54982 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:56:17:545.824 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:45508 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:56:17:590.976 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:45508 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504 2025-02-13 Thu 02:56:30:063.198 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738056972032, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 39 without seqno , total = 0 2025-02-13 Thu 02:56:30:063.301 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:56:30:063.428 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: SANDESH: Send FAILED: 1739415390063082 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 370 dequeues = 370 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 498 dequeues = 498 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739415361046474 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739415365045612 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:56:30:063.468 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:56:30:063.481 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:56:30:063.620 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: SANDESH: Send FAILED: 1739415390063175 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 90 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 241 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 370, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:56:30:063.641 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737679513344, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:56:31:046.798 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738719057664, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:56:31:046.899 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:56:31:046.958 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:56:31:047.065 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Idle 2025-02-13 Thu 02:56:31:047.158 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:56:31:047.190 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: SANDESH: Send FAILED: 1739415391047005 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:56:31:047.212 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:56:31:047.222 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:56:31:047.275 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:56:31:047.443 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: SANDESH: Send FAILED: 1739415391047146 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 90 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 243 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 372, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:56:31:047.462 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:57:00:063.916 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:57:00:064.095 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: SANDESH: Send FAILED: 1739415420063446 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 374 dequeues = 374 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 504 dequeues = 504 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739415391047063 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739415391047274 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:57:00:064.116 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:57:00:064.132 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:57:00:064.470 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: SANDESH: Send FAILED: 1739415420063855 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 91 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 244 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 374, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:57:00:064.510 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737650124544, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:57:01:047.220 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:57:01:047.661 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:57:01:047.844 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:57:01:047.892 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: SANDESH: Send FAILED: 1739415421047319 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:57:01:047.906 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:57:01:047.921 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:57:01:048.136 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: SANDESH: Send FAILED: 1739415421047821 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 91 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 39 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 246 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 376, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:57:01:048.162 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738044376832, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:57:05:046.230 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738719057664, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:57:05:046.327 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738719057664, Pid 1]: SANDESH: Send FAILED: 1739415425046162 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:57:05:046.340 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738719057664, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:57:30:064.469 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 40 without seqno , total = 0 2025-02-13 Thu 02:57:30:064.575 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:57:30:064.723 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739415450064343 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 379 dequeues = 379 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 510 dequeues = 510 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739415421047328 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739415425046211 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:57:30:064.742 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:57:30:064.756 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:57:30:064.928 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: SANDESH: Send FAILED: 1739415450064448 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 92 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 247 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 379, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:57:30:064.981 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:57:31:047.809 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737700497152, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:57:31:047.909 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:57:31:047.972 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:57:31:048.097 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Idle 2025-02-13 Thu 02:57:31:048.247 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:57:31:048.294 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: SANDESH: Send FAILED: 1739415451048022 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:57:31:048.309 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:57:31:048.324 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:57:31:048.339 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:57:31:048.587 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: SANDESH: Send FAILED: 1739415451048230 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 92 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 249 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 381, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:57:31:048.648 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738048575232, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:58:00:064.820 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:58:00:065.239 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: SANDESH: Send FAILED: 1739415480064670 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 383 dequeues = 383 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 44 dequeues = 44 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 516 dequeues = 516 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1739415451048094 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1739415451048337 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:58:00:065.258 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:58:00:065.270 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:58:00:065.425 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: SANDESH: Send FAILED: 1739415480064774 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 93 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 250 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 383, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:58:00:065.452 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738086360832, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:58:01:048.260 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737675314944, Pid 1]: Processing scm::EvIdleHoldTimerExpired in state Idle 2025-02-13 Thu 02:58:01:048.548 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737675314944, Pid 1]: Connect : Start Connect timer 10.0.0.17:8086 2025-02-13 Thu 02:58:01:048.670 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737675314944, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:58:01:048.724 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737675314944, Pid 1]: SANDESH: Send FAILED: 1739415481048360 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.17:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:58:01:048.745 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737675314944, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:58:01:048.755 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737675314944, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:58:01:048.922 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737675314944, Pid 1]: SANDESH: Send FAILED: 1739415481048656 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 93 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 252 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 385, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:58:01:048.950 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737675314944, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:58:05:047.172 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:58:05:047.569 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: SANDESH: Send FAILED: 1739415485047098 [SYS_NOTICE]: QEDbStatsUve: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 table_info= [ [ ] ] errors= [ write_tablespace_fails = 0 read_tablespace_fails = 0 write_table_fails = 0 read_table_fails = 0 write_column_fails = 0 write_batch_column_fails = 0 read_column_fails = 0 ] stats_info= [ [ ] ] cql_stats= [ requests_one_minute_rate = 0.28692 stats= [ total_connections = 2 available_connections = 2 exceeded_pending_requests_water_mark = 0 exceeded_write_bytes_water_mark = 0 ] errors= [ connection_timeouts = 0 pending_request_timeouts = 0 request_timeouts = 0 ] ] ] 2025-02-13 Thu 02:58:05:047.584 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: QEDbStatsUve 2025-02-13 Thu 02:58:30:065.326 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: SyncAllMaps for UserDefinedLogStatistic:2 period 60 cycle 41 without seqno , total = 0 2025-02-13 Thu 02:58:30:065.381 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:58:30:065.960 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: SANDESH: Send FAILED: 1739415510065224 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvConnectTimerExpired enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 44 dequeues = 44 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 388 dequeues = 388 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 44 dequeues = 44 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 522 dequeues = 522 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Connect state_since = 1739415481048369 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1739415485047152 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2025-02-13 Thu 02:58:30:065.995 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:58:30:066.008 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Processing scm::EvSandeshSend in state Connect 2025-02-13 Thu 02:58:30:066.187 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: SANDESH: Send FAILED: 1739415510065306 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 94 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 253 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 388, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:58:30:066.202 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:58:31:048.696 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738740909824, Pid 1]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:58:31:048.785 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Processing scm::EvConnectTimerExpired in state Connect 2025-02-13 Thu 02:58:31:048.820 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Connect : EvConnectTimerExpired 2025-02-13 Thu 02:58:31:048.908 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Idle 2025-02-13 Thu 02:58:31:049.023 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:58:31:049.056 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: SANDESH: Send FAILED: 1739415511048857 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1 process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvConnectTimerExpired ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.17:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.17:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-02-13 Thu 02:58:31:049.079 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE 2025-02-13 Thu 02:58:31:049.089 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Processing scm::EvTcpDeleteSession in state Idle 2025-02-13 Thu 02:58:31:049.127 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Processing scm::EvSandeshSend in state Idle 2025-02-13 Thu 02:58:31:049.291 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: SANDESH: Send FAILED: 1739415511049010 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-473-1:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1739412983753073 collector_name = collector_ip = 10.0.0.17:8086 collector_list= [ [ (*_iter40) = 10.0.0.17:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 94 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QEDbStatsUve [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 41 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 255 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 91 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 133, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 390, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-02-13 Thu 02:58:31:049.310 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139737666918144, Pid 1]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace 2025-02-13 Thu 02:58:39:841.737 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:32816 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256 2025-02-13 Thu 02:58:39:886.824 UTC cn-jenkins-deploy-platform-juju-os-473-1 [Thread 139738909763712, Pid 1]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8091::10.0.0.17:32816 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504