2025-03-31 Mon 02:07:29:801.426 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: SANDESH: No Client: 1743386849801324 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local:Control:contrail-dns:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8092 start_time = 1743386849801217 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:07:29:802.540 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980993709824, Pid 1]: SANDESH: Send FAILED: 1743386849802122 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local:Control:contrail-dns:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1 http_port = 8092 start_time = 1743386849801217 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:07:29:802.648 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980993709824, Pid 1]: SANDESH: Send FAILED: 1743386849802168 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local:Control:contrail-dns:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1 http_port = 8092 start_time = 1743386849801217 collector_name = collector_ip = 10.0.0.23:8086 collector_list= [ [ (*_iter40) = 10.0.0.23:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:07:29:802.722 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980993709824, Pid 1]: SANDESH: Send FAILED: 1743386849802445 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local:Control:contrail-dns:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8092 start_time = 1743386849801217 collector_name = collector_ip = 10.0.0.23:8086 collector_list= [ [ (*_iter40) = 10.0.0.23:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:07:30:349.989 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: Bulk sync has been started controller/src/dns/main.cc 403 2025-03-31 Mon 02:07:30:359.868 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980894623488, Pid 1]: SANDESH: Send FAILED: 1743386850359610 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Down description = RabbitMQ connection down ], ] ] flag_infos= [ [ ] ] description = Number of connections:2, Expected:3 Missing: Database:Cassandra ], ] ] ] 2025-03-31 Mon 02:07:30:446.094 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980882028288, Pid 1]: SANDESH: Send FAILED: 1743386850444112 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Number of connections:2, Expected:3 Missing: Database:Cassandra ], ] ] ] 2025-03-31 Mon 02:07:30:446.155 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980882028288, Pid 1]: SANDESH: Send FAILED: 1743386850444199 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.23:9041, ] ] status = Down description = Lost Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector, Database:Cassandra connection down ], ] ] ] 2025-03-31 Mon 02:07:30:574.468 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980877829888, Pid 1]: SANDESH: Send FAILED: 1743386850566743 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.23:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:07:30:588.656 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980890425088, Pid 1]: ConfigClientManager [SYS_WARN]: ConfigClientGetRowError: GetRow failed for table obj_fq_name_table access_control_list src/contrail-common/config-client-mgr/config_cassandra_client.cc 480 2025-03-31 Mon 02:07:30:592.737 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980989511424, Pid 1]: SANDESH: Send FAILED: 1743386850588590 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.23:9041, ] ] status = Down description = Lost Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector, Database:Cassandra connection down ], ] ] ] 2025-03-31 Mon 02:07:35:598.068 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980882028288, Pid 1]: SANDESH: Send FAILED: 1743386855597850 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.23:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:07:36:400.111 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980877829888, Pid 1]: SANDESH: Send FAILED: 1743386856399482 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.23:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:07:36:458.293 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: EndOfRib point was reached, IF-MAP objects have been created controller/src/dns/main.cc 86 2025-03-31 Mon 02:07:36:459.036 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.virtual_DNS.0, counting 0 elements controller/src/dns/main.cc 98 2025-03-31 Mon 02:07:36:460.018 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.virtual_DNS_record.0, counting 0 elements controller/src/dns/main.cc 98 2025-03-31 Mon 02:07:36:460.123 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.network_ipam.0, counting 0 elements controller/src/dns/main.cc 98 2025-03-31 Mon 02:07:36:460.141 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.virtual_network_network_ipam.0, counting 0 elements controller/src/dns/main.cc 98 2025-03-31 Mon 02:07:36:460.155 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.global_qos_config.0, counting 0 elements controller/src/dns/main.cc 98 2025-03-31 Mon 02:07:36:460.167 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: VnniConfig has been updated after the bulk sync, the number of elements: 0 controller/src/dns/main.cc 109 2025-03-31 Mon 02:07:36:460.180 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: IpamConfig has been updated after the bulk sync, the number of elements: 0 controller/src/dns/main.cc 120 2025-03-31 Mon 02:07:36:460.191 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: VirtualDnsConfig has been updated after the bulk sync, the number of elements: 0 controller/src/dns/main.cc 131 2025-03-31 Mon 02:07:36:460.202 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: VirtualDnsRecordConfig has been updated after the bulk sync, the number of elements: 0 controller/src/dns/main.cc 142 2025-03-31 Mon 02:07:36:460.214 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: GlobalQosConfig has been updated after the bulk sync controller/src/dns/main.cc 152 2025-03-31 Mon 02:07:36:468.149 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980976916224, Pid 1]: SANDESH: Send FAILED: 1743386856460291 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.23:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Down description = RabbitMQ connection down ], ] ] flag_infos= [ [ ] ] description = Collector, Database:RabbitMQ connection down ], ] ] ] 2025-03-31 Mon 02:07:36:824.911 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980877829888, Pid 1]: SANDESH: Send FAILED: 1743386856824077 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.23:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:07:36:841.744 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980890425088, Pid 1]: SANDESH: Send FAILED: 1743386856841590 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.23:8086, ] ] status = Initializing description = Connect : EvTcpConnected ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.23:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.23:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-03-31 Mon 02:07:36:842.632 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980877829888, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: BIND named down; DNS is not operational controller/src/dns/mgr/dns_mgr.cc 747 2025-03-31 Mon 02:07:36:856.338 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980890425088, Pid 1]: SANDESH: Send FAILED: 1743386856841703 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-155-1.cluster.local:Control:contrail-dns:0 client_info= [ status = ClientInit successful_connections = 1 pid = 1 http_port = 8092 start_time = 1743386849801217 collector_name = collector_ip = 10.0.0.23:8086 collector_list= [ [ (*_iter40) = 10.0.0.23:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ConfigClientGetRowError [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = ConfigClientMgrDebug [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 145 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = DnsOperational [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 12 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 9 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 162, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 12, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-03-31 Mon 02:07:39:530.677 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980882028288, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: BIND named down; DNS is not operational controller/src/dns/mgr/dns_mgr.cc 747 2025-03-31 Mon 02:07:42:206.641 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980890425088, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: BIND named down; DNS is not operational controller/src/dns/mgr/dns_mgr.cc 747 2025-03-31 Mon 02:07:43:411.030 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.23:8092::10.0.0.23:52776 > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2025-03-31 Mon 02:07:44:081.060 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.23:8092::10.0.0.23:52838 < Read failed due to error system 104 : Connection reset by peer src/contrail-common/io/tcp_session.cc 510 2025-03-31 Mon 02:07:44:081.117 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139981151774848, Pid 1]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.23:8092::10.0.0.23:52838 > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2025-03-31 Mon 02:07:44:619.677 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980886226688, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: BIND named down; DNS is not operational controller/src/dns/mgr/dns_mgr.cc 747 2025-03-31 Mon 02:08:04:167.349 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980993709824, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 6aa3ac58-7e41-470d-9ce0-47142882a321 controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:04:352.446 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980894623488, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 7109fb76-4f11-4571-9c76-1068fd48ac3b controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:04:630.722 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980894623488, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 0e15fbb4-7600-416b-9c71-ce475e40da43 controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:04:918.738 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980898821888, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 36ff792b-beb7-4b02-8f36-a23cde60d61c controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:05:139.931 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980894623488, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 4f19e372-311d-4dba-9b5d-5f1e5b185735 controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:05:501.237 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980903020288, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 3c2c41cf-1649-4004-b5f1-a665d27dbb5f controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:06:020.056 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980882028288, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 36ff792b-beb7-4b02-8f36-a23cde60d61c controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:06:020.159 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980882028288, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 4f19e372-311d-4dba-9b5d-5f1e5b185735 controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:06:332.990 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980886226688, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 3c2c41cf-1649-4004-b5f1-a665d27dbb5f controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:06:340.200 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980993709824, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 36ff792b-beb7-4b02-8f36-a23cde60d61c controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:06:434.947 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980903020288, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 3c2c41cf-1649-4004-b5f1-a665d27dbb5f controller/src/ifmap/client/config_json_parser.cc 223 2025-03-31 Mon 02:08:06:435.193 UTC cn-jenkins-deploy-platform-k8s-manifests-155-1 [Thread 139980886226688, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 4f19e372-311d-4dba-9b5d-5f1e5b185735 controller/src/ifmap/client/config_json_parser.cc 223