2025-03-31 Mon 02:29:10:259.117 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388150258549  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:29:10:259.138 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:29:10:259.150 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:29:10:259.426 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388150258842  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 42 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 96 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 23 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 436,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 153,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:29:10:259.446 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:29:18:686.410 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:49782 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2025-03-31 Mon 02:29:18:734.684 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:49782 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2025-03-31 Mon 02:29:39:244.849 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:29:39:244.975 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388179244711  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 17 dequeues = 17 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 155 dequeues = 155 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 211 dequeues = 211 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388150258556 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388150259148 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:29:39:245.022 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:29:39:245.034 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:29:39:245.296 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388179244804  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 97 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 25 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 438,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 155,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:29:39:245.315 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:29:40:258.854 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:29:40:258.941 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:29:40:258.978 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:29:40:259.088 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Idle
2025-03-31 Mon 02:29:40:259.194 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:29:40:259.222 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388180259036  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:29:40:259.231 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:29:40:259.239 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:29:40:259.251 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:29:40:259.522 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388180259181  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 43 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 99 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 25 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 438,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 157,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:29:40:259.552 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:29:49:036.055 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:56736 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2025-03-31 Mon 02:29:49:098.988 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:56736 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2025-03-31 Mon 02:30:09:245.122 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:30:09:245.226 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: SANDESH: Send FAILED: 1743388209244957  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 159 dequeues = 159 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 217 dequeues = 217 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388180259085 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388180259250 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:30:09:245.241 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:30:09:245.251 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:30:09:245.536 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: SANDESH: Send FAILED: 1743388209245074  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 100 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 159,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:30:09:245.576 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:30:10:259.245 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:30:10:259.447 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:30:10:259.552 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:30:10:259.581 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388210259323  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:30:10:259.594 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:30:10:259.603 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:30:10:259.966 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388210259539  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 102 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 161,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:30:10:259.992 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:30:39:245.392 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:30:39:245.537 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388239245249  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 18 dequeues = 18 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 163 dequeues = 163 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 222 dequeues = 222 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388210259330 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388210259601 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:30:39:245.557 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:30:39:245.579 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:30:39:245.920 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388239245337  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 103 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 163,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:30:39:245.950 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:30:40:259.585 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:30:40:259.672 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:30:40:259.713 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:30:40:259.859 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Idle
2025-03-31 Mon 02:30:40:259.961 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:30:40:260.000 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: SANDESH: Send FAILED: 1743388240259807  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:30:40:260.009 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:30:40:260.017 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:30:40:260.026 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:30:40:260.371 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: SANDESH: Send FAILED: 1743388240259948  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 45 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 105 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 165,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:30:40:260.397 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:31:09:245.636 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:31:09:245.766 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: SANDESH: Send FAILED: 1743388269245470  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 167 dequeues = 167 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 228 dequeues = 228 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388240259857 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388240260024 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:31:09:245.782 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:31:09:245.792 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:31:09:246.050 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: SANDESH: Send FAILED: 1743388269245591  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 106 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 167,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:31:09:246.081 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:31:10:260.001 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:31:10:260.279 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:31:10:260.412 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:31:10:260.445 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: SANDESH: Send FAILED: 1743388270260087  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:31:10:260.454 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:31:10:260.463 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:31:10:260.798 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: SANDESH: Send FAILED: 1743388270260393  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 46 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 108 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 169,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:31:10:260.835 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:31:39:245.914 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:31:39:246.022 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: SANDESH: Send FAILED: 1743388299245777  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 19 dequeues = 19 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 171 dequeues = 171 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 233 dequeues = 233 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388270260097 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388270260461 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:31:39:246.046 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:31:39:246.055 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:31:39:246.336 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: SANDESH: Send FAILED: 1743388299245884  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 109 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 171,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:31:39:246.356 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:31:40:260.417 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:31:40:260.479 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:31:40:260.492 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:31:40:260.780 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Idle
2025-03-31 Mon 02:31:40:260.883 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:31:40:260.911 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743388300260712  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:31:40:260.920 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:31:40:260.928 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:31:40:260.960 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:31:40:261.255 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743388300260871  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 47 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 111 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 173,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:31:40:262.080 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:32:09:246.152 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:32:09:246.291 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: SANDESH: Send FAILED: 1743388329246019  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 175 dequeues = 175 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 239 dequeues = 239 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388300260777 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388300260957 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:32:09:246.323 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:32:09:246.333 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:32:09:246.617 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: SANDESH: Send FAILED: 1743388329246127  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 112 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 175,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:32:09:246.638 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:32:10:260.923 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:32:10:261.238 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:32:10:261.349 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:32:10:261.377 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388330261075  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:32:10:261.386 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:32:10:261.395 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:32:10:261.656 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388330261334  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 48 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 114 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 177,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:32:10:262.306 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:32:39:246.442 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:32:39:246.565 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388359246302  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 20 dequeues = 20 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 179 dequeues = 179 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 244 dequeues = 244 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388330261083 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388330261393 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:32:39:246.593 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:32:39:246.604 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:32:39:246.897 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388359246396  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 115 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 179,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:32:39:246.929 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:32:40:261.976 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:32:40:262.037 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:32:40:262.233 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:32:40:262.322 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Idle
2025-03-31 Mon 02:32:40:262.422 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:32:40:262.450 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388360262269  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:32:40:262.469 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:32:40:262.481 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:32:40:262.505 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:32:40:262.773 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388360262410  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 49 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 117 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 181,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:32:40:262.794 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:33:09:246.745 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:33:09:246.886 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743388389246537  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 183 dequeues = 183 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 250 dequeues = 250 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388360262320 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388360262503 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:33:09:246.917 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:33:09:246.932 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:33:09:247.228 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743388389246678  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 50 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 118 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 183,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:33:09:247.264 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:33:10:262.482 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:33:10:262.685 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:33:10:262.800 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:33:10:262.841 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388390262563  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:33:10:262.850 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:33:10:262.859 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:33:10:263.129 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388390262786  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 50 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 120 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 185,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:33:10:263.164 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:33:39:247.176 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:33:39:247.326 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388419246898  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 21 dequeues = 21 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 187 dequeues = 187 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 255 dequeues = 255 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388390262569 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388390262857 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:33:39:247.344 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:33:39:247.355 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:33:39:247.619 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388419247077  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 121 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 187,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:33:39:247.640 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:33:40:262.862 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:33:40:262.945 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:33:40:262.982 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:33:40:263.071 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Idle
2025-03-31 Mon 02:33:40:263.179 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:33:40:263.223 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388420263016  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:33:40:263.232 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:33:40:263.241 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:33:40:263.267 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:33:40:263.557 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388420263166  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 51 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 123 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 189,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:33:40:263.585 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:34:09:247.467 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:34:09:247.570 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388449247221  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 191 dequeues = 191 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 261 dequeues = 261 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388420263070 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388420263266 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:34:09:247.598 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:34:09:247.608 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:34:09:247.900 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388449247417  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 124 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 191,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:34:09:247.921 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:34:10:263.209 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:34:10:263.492 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:34:10:263.612 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:34:10:263.651 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388450263289  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:34:10:263.663 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:34:10:263.674 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:34:10:263.963 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388450263593  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 52 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 126 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 440,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 193,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:34:10:263.995 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:34:18:689.509 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:58850 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2025-03-31 Mon 02:34:18:739.225 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:58850 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2025-03-31 Mon 02:34:39:248.050 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:34:39:248.191 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388479247930  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 22 dequeues = 22 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 195 dequeues = 195 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 266 dequeues = 266 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388450263296 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388450263672 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:34:39:248.207 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:34:39:248.217 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:34:39:248.516 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388479248020  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 53 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 127 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 29 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 442,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 195,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:34:39:248.545 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:34:40:263.625 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:34:40:263.706 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:34:40:263.749 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:34:40:264.262 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Idle
2025-03-31 Mon 02:34:40:264.370 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:34:40:264.399 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388480263792  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:34:40:264.417 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:34:40:264.427 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:34:40:264.459 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:34:40:264.711 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388480264357  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 53 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 129 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 29 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 442,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 197,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:34:40:264.741 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:34:49:037.246 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:53302 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2025-03-31 Mon 02:34:49:087.117 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:53302 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2025-03-31 Mon 02:35:09:248.296 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:35:09:248.392 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388509248156  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 199 dequeues = 199 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 272 dequeues = 272 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388480264259 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388480264457 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:35:09:248.405 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:35:09:248.414 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:35:09:248.715 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388509248267  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 130 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 199,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:35:09:248.780 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:35:10:264.438 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:35:10:264.671 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:35:10:264.788 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:35:10:264.817 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: SANDESH: Send FAILED: 1743388510264522  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:35:10:264.826 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:35:10:264.834 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:35:10:265.147 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: SANDESH: Send FAILED: 1743388510264772  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 132 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 201,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:35:10:265.897 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:35:39:248.580 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:35:39:248.745 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388539248451  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 23 dequeues = 23 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 203 dequeues = 203 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 277 dequeues = 277 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388510264530 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388510264833 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:35:39:248.771 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:35:39:248.782 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:35:39:249.061 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388539248534  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 133 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 203,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:35:39:249.080 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:35:40:264.833 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:35:40:264.915 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:35:40:264.954 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:35:40:265.077 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Idle
2025-03-31 Mon 02:35:40:265.186 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:35:40:265.214 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743388540265025  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:35:40:265.223 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:35:40:265.232 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:35:40:265.262 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:35:40:265.553 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743388540265171  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 55 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 135 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 205,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:35:40:265.585 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:36:09:248.950 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:36:09:249.131 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388569248690  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 207 dequeues = 207 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 25 dequeues = 25 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 283 dequeues = 283 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388540265075 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388540265260 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:36:09:249.176 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:36:09:249.192 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:36:09:249.662 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388569248857  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 56 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 136 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 207,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:36:09:249.691 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:36:10:265.208 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:36:10:265.415 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:36:10:265.517 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:36:10:265.545 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388570265285  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:36:10:265.553 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:36:10:265.562 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:36:10:265.895 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388570265503  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 56 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 138 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 209,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:36:10:265.936 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:36:39:249.248 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:36:39:249.376 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388599249116  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 24 dequeues = 24 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 25 dequeues = 25 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 211 dequeues = 211 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 25 dequeues = 25 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 288 dequeues = 288 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388570265291 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388570265560 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:36:39:249.398 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:36:39:249.408 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:36:39:249.756 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388599249202  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 57 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 139 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 211,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:36:39:250.552 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:36:40:265.546 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:36:40:265.629 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:36:40:265.666 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:36:40:265.759 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Idle
2025-03-31 Mon 02:36:40:265.869 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:36:40:265.897 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388600265699  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:36:40:265.916 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:36:40:265.924 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:36:40:265.933 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:36:40:266.200 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388600265846  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 57 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 141 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 213,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:36:40:266.222 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:37:09:249.592 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:37:09:249.708 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388629249350  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 25 dequeues = 25 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 25 dequeues = 25 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 215 dequeues = 215 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 26 dequeues = 26 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 294 dequeues = 294 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388600265758 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388600265931 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:37:09:249.738 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:37:09:249.753 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:37:09:250.019 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388629249502  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 58 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 142 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 215,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:37:09:250.049 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:37:10:265.917 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:37:10:266.174 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:37:10:266.296 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:37:10:266.327 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388630266002  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:37:10:266.336 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:37:10:266.345 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:37:10:266.596 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388630266283  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 58 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 144 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 217,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:37:10:266.617 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:37:39:249.830 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:37:39:249.952 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: SANDESH: Send FAILED: 1743388659249677  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 25 dequeues = 25 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 26 dequeues = 26 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 219 dequeues = 219 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 26 dequeues = 26 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 299 dequeues = 299 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388630266009 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388630266344 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:37:39:249.967 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:37:39:249.988 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:37:39:250.258 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: SANDESH: Send FAILED: 1743388659249785  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 59 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 145 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 219,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:37:39:250.277 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:37:40:266.348 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:37:40:266.432 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:37:40:266.471 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:37:40:266.558 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Idle
2025-03-31 Mon 02:37:40:266.660 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:37:40:266.687 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388660266505  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:37:40:266.698 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:37:40:266.709 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:37:40:266.767 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:37:40:267.022 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743388660266648  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 59 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 147 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 221,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:37:40:267.051 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:38:09:250.068 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:38:09:250.174 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: SANDESH: Send FAILED: 1743388689249915  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 26 dequeues = 26 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 26 dequeues = 26 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 223 dequeues = 223 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 27 dequeues = 27 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 305 dequeues = 305 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388660266556 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388660266765 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:38:09:250.205 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:38:09:250.215 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:38:09:250.477 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: SANDESH: Send FAILED: 1743388689250023  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 60 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 148 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 223,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:38:09:250.496 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207227631360, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:38:10:266.678 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:38:10:266.960 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:38:10:267.076 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:38:10:267.106 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: SANDESH: Send FAILED: 1743388690266782  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:38:10:267.116 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:38:10:267.124 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:38:10:267.391 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: SANDESH: Send FAILED: 1743388690267062  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 60 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 150 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 225,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:38:10:267.417 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:38:39:250.338 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:38:39:250.467 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388719250187  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 26 dequeues = 26 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 27 dequeues = 27 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 227 dequeues = 227 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 27 dequeues = 27 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 310 dequeues = 310 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388690266788 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388690267123 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:38:39:250.493 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:38:39:250.504 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:38:39:250.815 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388719250283  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 61 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 151 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 227,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:38:39:250.838 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:38:40:275.078 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:38:40:275.168 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:38:40:275.204 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:38:40:275.282 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Idle
2025-03-31 Mon 02:38:40:275.383 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:38:40:275.412 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388720275236  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:38:40:275.428 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:38:40:275.437 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:38:40:275.446 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:38:40:275.707 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388720275370  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 61 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 153 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 229,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:38:40:275.738 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:39:09:250.589 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:39:09:250.704 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388749250434  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 27 dequeues = 27 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 27 dequeues = 27 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 231 dequeues = 231 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 28 dequeues = 28 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 316 dequeues = 316 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388720275281 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388720275445 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:39:09:250.734 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:39:09:250.749 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:39:09:251.041 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388749250538  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 62 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 154 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 231,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:39:09:251.074 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:39:10:275.438 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:39:10:275.649 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:39:10:275.763 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:39:10:275.792 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388750275509  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:39:10:275.812 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:39:10:275.821 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:39:10:276.080 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388750275748  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 62 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 156 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 31 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 444,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 233,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:39:10:276.107 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:39:18:693.109 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:40758 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2025-03-31 Mon 02:39:18:738.714 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:40758 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2025-03-31 Mon 02:39:39:250.861 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:39:39:251.046 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388779250726  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 27 dequeues = 27 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 28 dequeues = 28 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 235 dequeues = 235 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 28 dequeues = 28 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 321 dequeues = 321 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388750275516 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388750275820 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:39:39:251.066 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:39:39:251.091 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:39:39:251.363 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388779250814  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 63 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 157 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 33 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 446,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 235,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:39:39:251.384 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:39:40:275.782 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:39:40:275.852 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:39:40:275.866 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:39:40:275.960 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Idle
2025-03-31 Mon 02:39:40:276.111 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:39:40:276.154 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: SANDESH: Send FAILED: 1743388780275905  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:39:40:276.168 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:39:40:276.181 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:39:40:276.193 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:39:40:276.674 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: SANDESH: Send FAILED: 1743388780276093  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 63 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 159 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 33 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 446,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 237,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:39:40:276.730 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:39:49:038.904 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:37606 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2025-03-31 Mon 02:39:49:083.541 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:37606 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2025-03-31 Mon 02:40:09:251.212 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:40:09:251.384 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388809251001  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 28 dequeues = 28 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 28 dequeues = 28 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 239 dequeues = 239 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 29 dequeues = 29 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 327 dequeues = 327 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388780275958 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388780276191 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:40:09:251.402 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:40:09:251.413 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:40:09:251.743 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388809251155  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 64 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 160 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 239,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:40:09:251.769 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:40:10:276.105 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:40:10:276.440 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:40:10:276.552 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:40:10:276.581 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: SANDESH: Send FAILED: 1743388810276240  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:40:10:276.589 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:40:10:276.598 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:40:10:276.923 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: SANDESH: Send FAILED: 1743388810276538  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 64 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 162 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 241,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:40:10:277.509 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:40:39:251.451 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:40:39:251.594 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: SANDESH: Send FAILED: 1743388839251317  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 28 dequeues = 28 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 29 dequeues = 29 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 243 dequeues = 243 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 29 dequeues = 29 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 332 dequeues = 332 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388810276249 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388810276596 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:40:39:251.614 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:40:39:251.625 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:40:39:251.969 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: SANDESH: Send FAILED: 1743388839251402  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 65 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 163 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 243,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:40:39:252.012 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:40:40:276.579 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:40:40:276.835 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:40:40:276.864 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:40:40:276.961 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Idle
2025-03-31 Mon 02:40:40:277.121 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:40:40:277.151 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388840276902  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:40:40:277.171 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:40:40:277.179 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:40:40:277.207 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:40:40:277.457 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: SANDESH: Send FAILED: 1743388840277106  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 65 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 165 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 245,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:40:40:278.072 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207473334016, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:41:09:251.726 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:41:09:251.843 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: SANDESH: Send FAILED: 1743388869251550  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 29 dequeues = 29 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 29 dequeues = 29 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 247 dequeues = 247 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 30 dequeues = 30 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 338 dequeues = 338 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388840276960 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388840277206 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:41:09:251.858 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:41:09:251.869 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:41:09:252.128 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: SANDESH: Send FAILED: 1743388869251666  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 66 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 166 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 247,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:41:09:252.156 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:41:10:277.138 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:41:10:277.399 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:41:10:277.519 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:41:10:277.548 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388870277244  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:41:10:277.557 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:41:10:277.566 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:41:10:277.856 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: SANDESH: Send FAILED: 1743388870277505  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 66 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 168 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 249,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:41:10:277.882 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:41:39:251.985 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:41:39:252.115 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: SANDESH: Send FAILED: 1743388899251861  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 29 dequeues = 29 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 30 dequeues = 30 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 251 dequeues = 251 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 30 dequeues = 30 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 343 dequeues = 343 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388870277253 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388870277564 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:41:39:252.140 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:41:39:252.153 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:41:39:252.524 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: SANDESH: Send FAILED: 1743388899251942  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 67 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 169 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 251,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:41:39:253.135 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207261218560, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:41:40:277.532 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:41:40:277.615 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:41:40:277.652 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:41:40:277.759 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Idle
2025-03-31 Mon 02:41:40:277.860 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:41:40:277.902 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: SANDESH: Send FAILED: 1743388900277687  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:41:40:277.911 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:41:40:277.920 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:41:40:277.929 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:41:40:278.192 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: SANDESH: Send FAILED: 1743388900277847  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 67 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 171 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 253,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:41:40:278.220 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:42:09:252.230 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:42:09:252.347 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388929252074  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 30 dequeues = 30 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 30 dequeues = 30 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 255 dequeues = 255 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 349 dequeues = 349 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388900277758 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388900277928 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:42:09:252.377 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:42:09:252.388 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:42:09:252.649 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743388929252183  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 68 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 172 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 255,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:42:09:252.683 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:42:10:277.949 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:42:10:278.196 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:42:10:278.303 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:42:10:278.331 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: SANDESH: Send FAILED: 1743388930278031  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:42:10:278.339 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:42:10:278.347 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:42:10:278.609 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: SANDESH: Send FAILED: 1743388930278289  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 68 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 174 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 257,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:42:10:278.636 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:42:39:252.486 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:42:39:252.653 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388959252356  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 30 dequeues = 30 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 259 dequeues = 259 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 354 dequeues = 354 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388930278040 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388930278346 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:42:39:252.669 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:42:39:252.679 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:42:39:253.013 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: SANDESH: Send FAILED: 1743388959252440  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 69 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 175 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 259,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:42:39:253.043 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:42:40:278.339 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207257020160, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:42:40:278.413 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:42:40:278.447 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:42:40:278.524 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Idle
2025-03-31 Mon 02:42:40:278.622 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:42:40:278.648 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: SANDESH: Send FAILED: 1743388960278479  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:42:40:278.667 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:42:40:278.675 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:42:40:278.708 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:42:40:279.112 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: SANDESH: Send FAILED: 1743388960278609  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 69 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 177 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 261,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:42:40:279.142 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:43:09:252.697 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:43:09:252.834 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743388989252554  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 263 dequeues = 263 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 360 dequeues = 360 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743388960278523 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743388960278706 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:43:09:252.851 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:43:09:252.861 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:43:09:253.143 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743388989252655  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 70 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 178 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 263,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:43:09:253.173 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:43:10:278.663 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:43:10:278.902 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:43:10:279.004 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:43:10:279.033 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: SANDESH: Send FAILED: 1743388990278758  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:43:10:279.052 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:43:10:279.060 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:43:10:279.311 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: SANDESH: Send FAILED: 1743388990278990  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 70 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 180 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 265,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:43:10:279.339 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207469135616, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:43:39:252.980 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:43:39:253.130 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743389019252839  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 267 dequeues = 267 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 365 dequeues = 365 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743388990278765 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743388990279059 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:43:39:253.145 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:43:39:253.155 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:43:39:253.439 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: SANDESH: Send FAILED: 1743389019252937  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 71 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 181 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 267,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:43:39:253.470 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207215036160, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:43:40:279.069 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:43:40:279.152 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:43:40:279.197 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:43:40:279.296 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Idle
2025-03-31 Mon 02:43:40:279.395 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:43:40:279.423 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: SANDESH: Send FAILED: 1743389020279239  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:43:40:279.432 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:43:40:279.440 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:43:40:279.470 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:43:40:279.767 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: SANDESH: Send FAILED: 1743389020279383  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 71 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 183 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 269,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:43:40:279.799 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:44:09:253.277 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:44:09:253.408 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: SANDESH: Send FAILED: 1743389049253073  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 271 dequeues = 271 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 371 dequeues = 371 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743389020279294 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743389020279468 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:44:09:253.438 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:44:09:253.449 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:44:09:253.783 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: SANDESH: Send FAILED: 1743389049253220  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 72 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 184 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 271,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:44:09:253.814 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207252821760, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:44:10:279.450 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:44:10:279.680 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:44:10:279.807 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:44:10:279.837 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: SANDESH: Send FAILED: 1743389050279532  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:44:10:279.846 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:44:10:279.854 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:44:10:280.128 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: SANDESH: Send FAILED: 1743389050279793  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 72 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 186 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 35 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 448,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 273,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:44:10:280.154 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207240226560, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:44:18:698.123 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:54540 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2025-03-31 Mon 02:44:18:743.302 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:54540 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2025-03-31 Mon 02:44:39:253.506 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:44:39:253.633 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: SANDESH: Send FAILED: 1743389079253380  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 275 dequeues = 275 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 376 dequeues = 376 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743389050279538 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743389050279853 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:44:39:253.647 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:44:39:253.657 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:44:39:253.930 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: SANDESH: Send FAILED: 1743389079253458  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 73 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 187 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 450,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 275,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:44:39:253.948 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:44:40:279.817 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:44:40:279.880 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:44:40:279.891 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:44:40:279.975 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Idle
2025-03-31 Mon 02:44:40:280.081 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:44:40:280.114 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743389080279920  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:44:40:280.136 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:44:40:280.149 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:44:40:280.187 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:44:40:280.448 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743389080280068  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 73 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 189 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 450,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 277,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:44:40:280.470 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:44:49:040.365 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:48464 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2025-03-31 Mon 02:44:49:091.635 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207643994240, Pid 1145270]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.17:8092::10.0.0.17:48464 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2025-03-31 Mon 02:45:09:253.741 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:45:09:253.849 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: SANDESH: Send FAILED: 1743389109253588  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 279 dequeues = 279 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 382 dequeues = 382 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743389080279974 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743389080280185 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:45:09:253.868 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:45:09:253.881 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:45:09:254.260 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: SANDESH: Send FAILED: 1743389109253696  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 74 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 190 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 39 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 452,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 279,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:45:09:255.275 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207248623360, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:45:10:280.158 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:45:10:280.371 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:45:10:280.477 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:45:10:280.507 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743389110280236  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:45:10:280.515 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:45:10:280.523 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:45:10:280.801 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: SANDESH: Send FAILED: 1743389110280463  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 74 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 192 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 39 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 452,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 281,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:45:10:280.829 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207231829760, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:45:39:254.008 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:45:39:254.157 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: SANDESH: Send FAILED: 1743389139253873  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 283 dequeues = 283 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 387 dequeues = 387 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743389110280245 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743389110280522 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:45:39:254.185 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:45:39:254.195 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:45:39:254.467 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: SANDESH: Send FAILED: 1743389139253979  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 75 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 193 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 39 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 452,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 283,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:45:39:254.488 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:45:40:280.512 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207236028160, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:45:40:280.593 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:45:40:280.627 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:45:40:280.736 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Idle
2025-03-31 Mon 02:45:40:280.834 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:45:40:280.862 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743389140280660  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:45:40:280.870 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:45:40:280.879 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:45:40:280.907 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:45:40:281.195 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: SANDESH: Send FAILED: 1743389140280822  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 75 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 195 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 39 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 452,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 285,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:45:40:281.224 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207485929216, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:46:09:254.268 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:46:09:254.366 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: SANDESH: Send FAILED: 1743389169254104  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 287 dequeues = 287 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 393 dequeues = 393 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1743389140280734 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1743389140280905 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:46:09:254.399 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:46:09:254.409 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:46:09:254.692 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: SANDESH: Send FAILED: 1743389169254220  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 76 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 196 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 39 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 452,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 287,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:46:09:254.730 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207244424960, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:46:10:280.860 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvIdleHoldTimerExpired in state Idle
2025-03-31 Mon 02:46:10:281.098 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Connect : Start Connect timer 10.0.0.17:8086
2025-03-31 Mon 02:46:10:281.209 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:46:10:281.243 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: SANDESH: Send FAILED: 1743389170280938  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.17:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:46:10:281.255 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:46:10:281.267 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:46:10:281.591 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: SANDESH: Send FAILED: 1743389170281195  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 76 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 198 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 39 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 452,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 289,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:46:10:281.622 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207219234560, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:46:39:254.484 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:46:39:254.613 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: SANDESH: Send FAILED: 1743389199254376  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 291 dequeues = 291 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 398 dequeues = 398 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1743389170280945 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1743389170281264 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2025-03-31 Mon 02:46:39:254.631 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:46:39:254.643 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvSandeshSend in state Connect
2025-03-31 Mon 02:46:39:254.967 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: SANDESH: Send FAILED: 1743389199254460  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 77 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 199 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 39 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 452,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 291,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2025-03-31 Mon 02:46:39:254.990 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2025-03-31 Mon 02:46:40:281.247 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207477532416, Pid 1145270]: 10.0.0.17:8086 EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:46:40:281.321 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvConnectTimerExpired in state Connect
2025-03-31 Mon 02:46:40:281.354 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Connect : EvConnectTimerExpired
2025-03-31 Mon 02:46:40:281.433 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Idle
2025-03-31 Mon 02:46:40:281.532 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:46:40:281.560 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: SANDESH: Send FAILED: 1743389200281387  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.17:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.17:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2025-03-31 Mon 02:46:40:281.576 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2025-03-31 Mon 02:46:40:281.584 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvTcpDeleteSession in state Idle
2025-03-31 Mon 02:46:40:281.593 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: Processing scm::EvSandeshSend in state Idle
2025-03-31 Mon 02:46:40:281.868 UTC  cn-jenkins-deploy-platform-juju-os-494-1 [Thread 140207481730816, Pid 1145270]: SANDESH: Send FAILED: 1743389200281519  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-494-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1145270 http_port = 8092 start_time = 1743387253208277 collector_name =  collector_ip = 10.0.0.17:8086 collector_list= [  [ (*_iter40) = 10.0.0.17:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 243 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 4 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 68 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 77 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 201 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 39 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 452,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 293,  _iter59->first = sent _iter59->second = 0,  ] ] ]